diff --git a/glance/api/v2/image_data.py b/glance/api/v2/image_data.py index 598f481d9b..cfbea0af80 100644 --- a/glance/api/v2/image_data.py +++ b/glance/api/v2/image_data.py @@ -169,6 +169,8 @@ class ImageDataController(object): encodeutils.exception_to_unicode(e)) image_repo.save(image, from_state='queued') + ks_quota.enforce_image_count_uploading(req.context, + req.context.owner) image.set_data(data, size, backend=backend) try: @@ -275,6 +277,12 @@ class ImageDataController(object): raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req) + except exception.LimitExceeded as e: + LOG.error(str(e)) + self._restore(image_repo, image) + raise webob.exc.HTTPRequestEntityTooLarge(explanation=str(e), + request=req) + except glance_store.StorageWriteDenied as e: msg = _("Insufficient permissions on image " "storage media: %s") % encodeutils.exception_to_unicode(e) @@ -357,6 +365,8 @@ class ImageDataController(object): image = image_repo.get(image_id) image.status = 'uploading' image_repo.save(image, from_state='queued') + ks_quota.enforce_image_count_uploading(req.context, + req.context.owner) try: uri, size, id, store_info = staging_store.add( image_id, utils.LimitingReader( @@ -401,6 +411,12 @@ class ImageDataController(object): raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req) + except exception.LimitExceeded as e: + LOG.debug(str(e)) + self._unstage(image_repo, image, staging_store) + raise webob.exc.HTTPRequestEntityTooLarge(explanation=str(e), + request=req) + except glance_store.StorageWriteDenied as e: msg = _("Insufficient permissions on image " "storage media: %s") % encodeutils.exception_to_unicode(e) diff --git a/glance/api/v2/images.py b/glance/api/v2/images.py index 16a93ea89a..68ff7193bc 100644 --- a/glance/api/v2/images.py +++ b/glance/api/v2/images.py @@ -469,6 +469,9 @@ class ImagesController(object): raise webob.exc.HTTPConflict(explanation=e.msg) except exception.InvalidImageStatusTransition as e: raise webob.exc.HTTPConflict(explanation=e.msg) + except exception.LimitExceeded as e: + raise webob.exc.HTTPRequestEntityTooLarge(explanation=str(e), + request=req) except ValueError as e: LOG.debug("Cannot import data for image %(id)s: %(e)s", {'id': image_id, diff --git a/glance/async_/flows/api_image_import.py b/glance/async_/flows/api_image_import.py index e8056699f7..fa362c8529 100644 --- a/glance/async_/flows/api_image_import.py +++ b/glance/async_/flows/api_image_import.py @@ -882,5 +882,8 @@ def get_flow(**kwargs): stores, action_wrapper, ks_quota.enforce_image_staging_total, delta=image_size) + assert_quota(kwargs['context'], task_repo, task_id, + stores, action_wrapper, + ks_quota.enforce_image_count_uploading) return flow diff --git a/glance/async_/taskflow_executor.py b/glance/async_/taskflow_executor.py index e8adbcc305..0951d5a7dc 100644 --- a/glance/async_/taskflow_executor.py +++ b/glance/async_/taskflow_executor.py @@ -137,6 +137,8 @@ class TaskExecutor(glance.async_.TaskExecutor): raise exception.ImportTaskError(message=exc.reason) except (exception.BadStoreUri, exception.Invalid) as exc: raise exception.ImportTaskError(message=exc.msg) + except exception.LimitExceeded as exc: + raise exception.ImportTaskError(message=exc.msg) except RuntimeError: raise NotImplementedError() except Exception as e: diff --git a/glance/quota/keystone.py b/glance/quota/keystone.py index 7f6cdd2304..fa8e8c3e14 100644 --- a/glance/quota/keystone.py +++ b/glance/quota/keystone.py @@ -30,6 +30,7 @@ limit.opts.register_opts(CONF) QUOTA_IMAGE_SIZE_TOTAL = 'image_size_total' QUOTA_IMAGE_STAGING_TOTAL = 'image_stage_total' QUOTA_IMAGE_COUNT_TOTAL = 'image_count_total' +QUOTA_IMAGE_COUNT_UPLOADING = 'image_count_uploading' def _enforce_some(context, project_id, quota_value_fns, deltas): @@ -125,3 +126,19 @@ def enforce_image_count_total(context, project_id): context, project_id, QUOTA_IMAGE_COUNT_TOTAL, lambda: db.user_get_image_count(context, project_id), delta=1) + + +def enforce_image_count_uploading(context, project_id): + """Enforce the image_count_uploading quota. + + This enforces the total count of images in any state of upload by + the supplied project_id. + + :param delta: This defaults to one, but should be zero when checking + an operation on an image that already counts against this + quota (i.e. a stage operation of an existing queue image). + """ + _enforce_one( + context, project_id, QUOTA_IMAGE_COUNT_UPLOADING, + lambda: db.user_get_uploading_count(context, project_id), + delta=0) diff --git a/glance/tests/functional/v2/test_images.py b/glance/tests/functional/v2/test_images.py index e1aee139f9..9542fe0f3c 100644 --- a/glance/tests/functional/v2/test_images.py +++ b/glance/tests/functional/v2/test_images.py @@ -7074,7 +7074,8 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase): def test_upload(self): # Set a quota of 5MiB self.set_limit({'image_size_total': 5, - 'image_count_total': 10}) + 'image_count_total': 10, + 'image_count_uploading': 10}) self.start_server() # First upload of 3MiB is good @@ -7098,7 +7099,8 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase): def test_import(self): # Set a quota of 5MiB self.set_limit({'image_size_total': 5, - 'image_count_total': 10}) + 'image_count_total': 10, + 'image_count_uploading': 10}) self.start_server() # First upload of 3MiB is good @@ -7121,7 +7123,8 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase): def test_import_would_go_over(self): # Set a quota limit of 5MiB self.set_limit({'image_size_total': 5, - 'image_count_total': 10}) + 'image_count_total': 10, + 'image_count_uploading': 10}) self.start_server() # First upload of 3MiB is good @@ -7161,7 +7164,8 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase): # Set a size quota of 5MiB, with more staging quota than we need. self.set_limit({'image_size_total': 5, 'image_count_total': 10, - 'image_stage_total': 15}) + 'image_stage_total': 15, + 'image_count_uploading': 10}) self.start_server() # First import of 3MiB is good @@ -7185,7 +7189,9 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase): # before copy. This request should succeed, but the copy task # should fail the staging quota check. self.set_limit({'image_size_total': 15, - 'image_stage_total': 5}) + 'image_count_total': 10, + 'image_stage_total': 5, + 'image_count_uploading': 10}) req = self._import_copy(image_id, ['store3']) self.assertEqual(202, req.status_code) self._wait_for_import(image_id) @@ -7193,7 +7199,9 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase): # If we increase our stage quota, we should now be able to copy. self.set_limit({'image_size_total': 15, - 'image_stage_total': 10}) + 'image_count_total': 10, + 'image_stage_total': 10, + 'image_count_uploading': 10}) req = self._import_copy(image_id, ['store3']) self.assertEqual(202, req.status_code) self._wait_for_import(image_id) @@ -7203,7 +7211,8 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase): # Set a quota of 5MiB self.set_limit({'image_size_total': 15, 'image_stage_total': 5, - 'image_count_total': 10}) + 'image_count_total': 10, + 'image_count_uploading': 10}) self.start_server() # Stage 6MiB, which is allowed to complete, but leaves us over @@ -7237,7 +7246,8 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase): def test_create(self): # Set a quota of 2 images self.set_limit({'image_size_total': 15, - 'image_count_total': 2}) + 'image_count_total': 2, + 'image_count_uploading': 10}) self.start_server() # Create one image @@ -7255,3 +7265,67 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase): # Now we can create that third image self._create() + + def test_uploading_methods(self): + self.set_limit({'image_size_total': 100, + 'image_stage_total': 100, + 'image_count_total': 100, + 'image_count_uploading': 1}) + self.start_server() + + # Create and stage one image. We are now at quota for count_uploading. + image_id = self._create_and_stage() + + # Make sure we can not stage any more images. + self._create_and_stage(expected_code=413) + + # Make sure we can not upload any more images. + self._create_and_upload(expected_code=413) + + # Finish importing one of the images, which should put us under quota + # for count_uploading. + resp = self._import_direct(image_id, ['store1']) + self.assertEqual(202, resp.status_code) + self.assertEqual('active', self._wait_for_import(image_id)['status']) + + # Make sure we can upload now. + self._create_and_upload() + + # Stage another, which should put us at quota for count_uploading. + image_id2 = self._create_and_stage() + + # Start a copy. The request should succeed (because async) but + # the task should ultimately fail because we are over quota. + # NOTE(danms): It would be nice to try to do another copy or + # upload while this is running, but since the task is fully + # async and the copy happens quickly, we can't really time it + # to avoid an unstable test (without some mocking). + resp = self._import_copy(image_id, ['store2']) + self.assertEqual(202, resp.status_code) + self._wait_for_import(image_id) + task = self._get_latest_task(image_id) + self.assertEqual('failure', task['status']) + self.assertIn('Resource image_count_uploading is over limit', + task['message']) + + # Finish the staged import. + self._import_direct(image_id2, ['store1']) + self.assertEqual(202, resp.status_code) + self._wait_for_import(image_id2) + + # Make sure we can upload again after the import finishes. + self._create_and_upload() + + # Re-try the copy that should now succeed and wait for it to + # finish. + resp = self._import_copy(image_id, ['store2']) + self.assertEqual(202, resp.status_code) + self._wait_for_import(image_id) + task = self._get_latest_task(image_id) + self.assertEqual('success', task['status']) + + # Make sure we can still upload. + self._create_and_upload() + + # Make sure we can still import. + self._create_and_import(stores=['store1'])