From ffc8b684b6ff24833283a245d51bd70db906c8e8 Mon Sep 17 00:00:00 2001 From: miaohb Date: Mon, 15 Jan 2018 20:22:07 -0800 Subject: [PATCH] Wait for volume creation If users create a container with new cinder volumes and the volumes are 'creating'. Zun should wait for the volume creation before proceeding the container creation process. Change-Id: I423a69a97271a2acdd73a85cccddb65b052d169e Closes-Bug: #1740942 --- zun/compute/manager.py | 17 ++++++++++ zun/container/docker/driver.py | 6 ++++ zun/container/driver.py | 3 ++ .../unit/compute/test_compute_manager.py | 33 ++++++++++++++----- zun/volume/driver.py | 12 ++++++- 5 files changed, 62 insertions(+), 9 deletions(-) diff --git a/zun/compute/manager.py b/zun/compute/manager.py index ea3bd4733..e85caaa6b 100644 --- a/zun/compute/manager.py +++ b/zun/compute/manager.py @@ -13,6 +13,7 @@ # under the License. import six +import time from oslo_log import log as logging from oslo_service import periodic_task @@ -129,10 +130,26 @@ class Manager(periodic_task.PeriodicTasks): container.host = None container.save(context) + def _wait_for_volumes_available(self, context, volumes, timeout=60, + poll_interval=1): + count = 0 + start_time = time.time() + while time.time() - start_time < timeout: + if count == len(volumes): + break + for vol in volumes: + if self.driver.is_volume_available(context, vol): + count = count + 1 + time.sleep(poll_interval) + else: + self.fail("Volumes did not reach available status after %d s" + % (timeout)) + def container_create(self, context, limits, requested_networks, requested_volumes, container, run, pci_requests=None): @utils.synchronized(container.uuid) def do_container_create(): + self._wait_for_volumes_available(context, requested_volumes) if not self._attach_volumes(context, container, requested_volumes): return created_container = self._do_container_create( diff --git a/zun/container/docker/driver.py b/zun/container/docker/driver.py index a64d60f66..5412c23d8 100644 --- a/zun/container/docker/driver.py +++ b/zun/container/docker/driver.py @@ -798,6 +798,12 @@ class DockerDriver(driver.ContainerDriver): context=context) volume_driver.delete(volume_mapping) + def is_volume_available(self, context, volume_mapping): + volume_driver = vol_driver.driver( + provider=volume_mapping.volume_provider, + context=context) + return volume_driver.is_volume_available(volume_mapping) + def _get_or_create_docker_network(self, context, network_api, neutron_net_id): docker_net_name = self._get_docker_network_name(context, diff --git a/zun/container/driver.py b/zun/container/driver.py index 512d83055..aeaa123e7 100644 --- a/zun/container/driver.py +++ b/zun/container/driver.py @@ -204,6 +204,9 @@ class ContainerDriver(object): def delete_volume(self, context, volume_mapping): raise NotImplementedError() + def is_volume_available(self, context, volume_mapping): + raise NotImplementedError() + def add_security_group(self, context, container, security_group, **kwargs): raise NotImplementedError() diff --git a/zun/tests/unit/compute/test_compute_manager.py b/zun/tests/unit/compute/test_compute_manager.py index 94ff70c45..2bfa73579 100644 --- a/zun/tests/unit/compute/test_compute_manager.py +++ b/zun/tests/unit/compute/test_compute_manager.py @@ -256,10 +256,12 @@ class TestManager(base.TestCase): @mock.patch('zun.image.driver.pull_image') @mock.patch.object(fake_driver, 'detach_volume') @mock.patch.object(fake_driver, 'attach_volume') + @mock.patch.object(fake_driver, 'is_volume_available') @mock.patch.object(fake_driver, 'create') @mock.patch.object(fake_driver, 'start') def test_container_run( - self, mock_start, mock_create, mock_attach_volume, + self, mock_start, mock_create, + mock_is_volume_available, mock_attach_volume, mock_detach_volume, mock_pull, mock_list_by_container, mock_save, mock_spawn_n): container = Container(self.context, **utils.get_test_container()) @@ -285,6 +287,7 @@ class TestManager(base.TestCase): mock_start.assert_called_once_with(self.context, container) mock_attach_volume.assert_called_once() mock_detach_volume.assert_not_called() + mock_is_volume_available.assert_called_once() self.assertEqual(1, len(FakeVolumeMapping.volumes)) @mock.patch('zun.common.utils.spawn_n') @@ -294,12 +297,15 @@ class TestManager(base.TestCase): @mock.patch('zun.image.driver.pull_image') @mock.patch.object(fake_driver, 'detach_volume') @mock.patch.object(fake_driver, 'attach_volume') + @mock.patch.object(fake_driver, 'is_volume_available') @mock.patch.object(fake_driver, 'create') @mock.patch.object(fake_driver, 'start') def test_container_run_driver_attach_failed( - self, mock_start, mock_create, mock_attach_volume, + self, mock_start, mock_create, + mock_is_volume_available, mock_attach_volume, mock_detach_volume, mock_pull, mock_list_by_container, mock_save, mock_spawn_n): + mock_is_volume_available.return_value = True mock_attach_volume.side_effect = [None, base.TestingException("fake")] container = Container(self.context, **utils.get_test_container()) vol = FakeVolumeMapping() @@ -334,9 +340,11 @@ class TestManager(base.TestCase): side_effect=FakeVolumeMapping.list_by_container) @mock.patch.object(fake_driver, 'detach_volume') @mock.patch.object(fake_driver, 'attach_volume') + @mock.patch.object(fake_driver, 'is_volume_available') @mock.patch('zun.image.driver.pull_image') def test_container_run_image_not_found( - self, mock_pull, mock_attach_volume, mock_detach_volume, + self, mock_pull, mock_is_volume_available, + mock_attach_volume, mock_detach_volume, mock_list_by_container, mock_save, mock_spawn_n): container_dict = utils.get_test_container( image='test:latest', image_driver='docker', @@ -360,6 +368,7 @@ class TestManager(base.TestCase): 'ifnotpresent', 'docker') mock_attach_volume.assert_called_once() mock_detach_volume.assert_called_once() + mock_is_volume_available.assert_called_once() self.assertEqual(0, len(FakeVolumeMapping.volumes)) @mock.patch('zun.common.utils.spawn_n') @@ -368,9 +377,11 @@ class TestManager(base.TestCase): side_effect=FakeVolumeMapping.list_by_container) @mock.patch.object(fake_driver, 'detach_volume') @mock.patch.object(fake_driver, 'attach_volume') + @mock.patch.object(fake_driver, 'is_volume_available') @mock.patch('zun.image.driver.pull_image') def test_container_run_image_pull_exception_raised( - self, mock_pull, mock_attach_volume, mock_detach_volume, + self, mock_pull, mock_is_volume_available, + mock_attach_volume, mock_detach_volume, mock_list_by_container, mock_save, mock_spawn_n): container_dict = utils.get_test_container( image='test:latest', image_driver='docker', @@ -394,6 +405,7 @@ class TestManager(base.TestCase): 'ifnotpresent', 'docker') mock_attach_volume.assert_called_once() mock_detach_volume.assert_called_once() + mock_is_volume_available.assert_called_once() self.assertEqual(0, len(FakeVolumeMapping.volumes)) @mock.patch('zun.common.utils.spawn_n') @@ -402,9 +414,11 @@ class TestManager(base.TestCase): side_effect=FakeVolumeMapping.list_by_container) @mock.patch.object(fake_driver, 'detach_volume') @mock.patch.object(fake_driver, 'attach_volume') + @mock.patch.object(fake_driver, 'is_volume_available') @mock.patch('zun.image.driver.pull_image') def test_container_run_image_pull_docker_error( - self, mock_pull, mock_attach_volume, mock_detach_volume, + self, mock_pull, mock_is_volume_available, + mock_attach_volume, mock_detach_volume, mock_list_by_container, mock_save, mock_spawn_n): container_dict = utils.get_test_container( image='test:latest', image_driver='docker', @@ -428,6 +442,7 @@ class TestManager(base.TestCase): 'ifnotpresent', 'docker') mock_attach_volume.assert_called_once() mock_detach_volume.assert_called_once() + mock_is_volume_available.assert_called_once() self.assertEqual(0, len(FakeVolumeMapping.volumes)) @mock.patch('zun.common.utils.spawn_n') @@ -436,12 +451,13 @@ class TestManager(base.TestCase): side_effect=FakeVolumeMapping.list_by_container) @mock.patch.object(fake_driver, 'detach_volume') @mock.patch.object(fake_driver, 'attach_volume') + @mock.patch.object(fake_driver, 'is_volume_available') @mock.patch('zun.image.driver.pull_image') @mock.patch.object(fake_driver, 'create') def test_container_run_create_raises_docker_error( - self, mock_create, mock_pull, mock_attach_volume, - mock_detach_volume, mock_list_by_container, mock_save, - mock_spawn_n): + self, mock_create, mock_pull, mock_is_volume_available, + mock_attach_volume, mock_detach_volume, + mock_list_by_container, mock_save, mock_spawn_n): container = Container(self.context, **utils.get_test_container()) image = {'image': 'repo', 'path': 'out_path', 'driver': 'glance', 'repo': 'test', 'tag': 'testtag'} @@ -467,6 +483,7 @@ class TestManager(base.TestCase): self.context, container, image, networks, volumes) mock_attach_volume.assert_called_once() mock_detach_volume.assert_called_once() + mock_is_volume_available.assert_called_once() self.assertEqual(0, len(FakeVolumeMapping.volumes)) @mock.patch.object(FakeResourceTracker, diff --git a/zun/volume/driver.py b/zun/volume/driver.py index f0f401668..0cbc9d736 100644 --- a/zun/volume/driver.py +++ b/zun/volume/driver.py @@ -23,9 +23,9 @@ from zun.common import exception from zun.common.i18n import _ from zun.common import mount import zun.conf +from zun.volume import cinder_api from zun.volume import cinder_workflow - LOG = logging.getLogger(__name__) CONF = zun.conf.CONF @@ -72,6 +72,9 @@ class VolumeDriver(object): def bind_mount(self, *args, **kwargs): raise NotImplementedError() + def is_volume_available(self, *args, **kwargs): + raise NotImplementedError() + class Cinder(VolumeDriver): @@ -116,3 +119,10 @@ class Cinder(VolumeDriver): def bind_mount(self, volume): mountpoint = mount.get_mountpoint(volume.volume_id) return mountpoint, volume.container_path + + def is_volume_available(self, volume): + ca = cinder_api.CinderAPI(self.context) + if 'available' == ca.get(volume.volume_id).status: + return True + else: + return False