Wait for volume creation

If users create a container with new cinder volumes and the volumes
are 'creating'. Zun should wait for the volume creation before
proceeding the container creation process.

Change-Id: I423a69a97271a2acdd73a85cccddb65b052d169e
Closes-Bug: #1740942

zun/compute/manager.py

@@ -13,6 +13,7 @@
 #    under the License.
 
 import six
+import time
 
 from oslo_log import log as logging
 from oslo_service import periodic_task
@@ -129,10 +130,26 @@ class Manager(periodic_task.PeriodicTasks):
             container.host = None
         container.save(context)
 
+    def _wait_for_volumes_available(self, context, volumes, timeout=60,
+                                    poll_interval=1):
+        count = 0
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            if count == len(volumes):
+                break
+            for vol in volumes:
+                if self.driver.is_volume_available(context, vol):
+                    count = count + 1
+            time.sleep(poll_interval)
+        else:
+            self.fail("Volumes did not reach available status after %d s"
+                      % (timeout))
+
     def container_create(self, context, limits, requested_networks,
                          requested_volumes, container, run, pci_requests=None):
         @utils.synchronized(container.uuid)
         def do_container_create():
+            self._wait_for_volumes_available(context, requested_volumes)
             if not self._attach_volumes(context, container, requested_volumes):
                 return
             created_container = self._do_container_create(

zun/container/docker/driver.py

@@ -798,6 +798,12 @@ class DockerDriver(driver.ContainerDriver):
             context=context)
         volume_driver.delete(volume_mapping)
 
+    def is_volume_available(self, context, volume_mapping):
+        volume_driver = vol_driver.driver(
+            provider=volume_mapping.volume_provider,
+            context=context)
+        return volume_driver.is_volume_available(volume_mapping)
+
     def _get_or_create_docker_network(self, context, network_api,
                                       neutron_net_id):
         docker_net_name = self._get_docker_network_name(context,

zun/container/driver.py

@@ -204,6 +204,9 @@ class ContainerDriver(object):
     def delete_volume(self, context, volume_mapping):
         raise NotImplementedError()
 
+    def is_volume_available(self, context, volume_mapping):
+        raise NotImplementedError()
+
     def add_security_group(self, context, container, security_group, **kwargs):
         raise NotImplementedError()
 

zun/tests/unit/compute/test_compute_manager.py

@@ -256,10 +256,12 @@ class TestManager(base.TestCase):
     @mock.patch('zun.image.driver.pull_image')
     @mock.patch.object(fake_driver, 'detach_volume')
     @mock.patch.object(fake_driver, 'attach_volume')
+    @mock.patch.object(fake_driver, 'is_volume_available')
     @mock.patch.object(fake_driver, 'create')
     @mock.patch.object(fake_driver, 'start')
     def test_container_run(
-            self, mock_start, mock_create, mock_attach_volume,
+            self, mock_start, mock_create,
+            mock_is_volume_available, mock_attach_volume,
             mock_detach_volume, mock_pull, mock_list_by_container, mock_save,
             mock_spawn_n):
         container = Container(self.context, **utils.get_test_container())
@@ -285,6 +287,7 @@ class TestManager(base.TestCase):
         mock_start.assert_called_once_with(self.context, container)
         mock_attach_volume.assert_called_once()
         mock_detach_volume.assert_not_called()
+        mock_is_volume_available.assert_called_once()
         self.assertEqual(1, len(FakeVolumeMapping.volumes))
 
     @mock.patch('zun.common.utils.spawn_n')
@@ -294,12 +297,15 @@ class TestManager(base.TestCase):
     @mock.patch('zun.image.driver.pull_image')
     @mock.patch.object(fake_driver, 'detach_volume')
     @mock.patch.object(fake_driver, 'attach_volume')
+    @mock.patch.object(fake_driver, 'is_volume_available')
     @mock.patch.object(fake_driver, 'create')
     @mock.patch.object(fake_driver, 'start')
     def test_container_run_driver_attach_failed(
-            self, mock_start, mock_create, mock_attach_volume,
+            self, mock_start, mock_create,
+            mock_is_volume_available, mock_attach_volume,
             mock_detach_volume, mock_pull, mock_list_by_container, mock_save,
             mock_spawn_n):
+        mock_is_volume_available.return_value = True
         mock_attach_volume.side_effect = [None, base.TestingException("fake")]
         container = Container(self.context, **utils.get_test_container())
         vol = FakeVolumeMapping()
@@ -334,9 +340,11 @@ class TestManager(base.TestCase):
                        side_effect=FakeVolumeMapping.list_by_container)
     @mock.patch.object(fake_driver, 'detach_volume')
     @mock.patch.object(fake_driver, 'attach_volume')
+    @mock.patch.object(fake_driver, 'is_volume_available')
     @mock.patch('zun.image.driver.pull_image')
     def test_container_run_image_not_found(
-            self, mock_pull, mock_attach_volume, mock_detach_volume,
+            self, mock_pull, mock_is_volume_available,
+            mock_attach_volume, mock_detach_volume,
             mock_list_by_container, mock_save, mock_spawn_n):
         container_dict = utils.get_test_container(
             image='test:latest', image_driver='docker',
@@ -360,6 +368,7 @@ class TestManager(base.TestCase):
                                           'ifnotpresent', 'docker')
         mock_attach_volume.assert_called_once()
         mock_detach_volume.assert_called_once()
+        mock_is_volume_available.assert_called_once()
         self.assertEqual(0, len(FakeVolumeMapping.volumes))
 
     @mock.patch('zun.common.utils.spawn_n')
@@ -368,9 +377,11 @@ class TestManager(base.TestCase):
                        side_effect=FakeVolumeMapping.list_by_container)
     @mock.patch.object(fake_driver, 'detach_volume')
     @mock.patch.object(fake_driver, 'attach_volume')
+    @mock.patch.object(fake_driver, 'is_volume_available')
     @mock.patch('zun.image.driver.pull_image')
     def test_container_run_image_pull_exception_raised(
-            self, mock_pull, mock_attach_volume, mock_detach_volume,
+            self, mock_pull, mock_is_volume_available,
+            mock_attach_volume, mock_detach_volume,
             mock_list_by_container, mock_save, mock_spawn_n):
         container_dict = utils.get_test_container(
             image='test:latest', image_driver='docker',
@@ -394,6 +405,7 @@ class TestManager(base.TestCase):
                                           'ifnotpresent', 'docker')
         mock_attach_volume.assert_called_once()
         mock_detach_volume.assert_called_once()
+        mock_is_volume_available.assert_called_once()
         self.assertEqual(0, len(FakeVolumeMapping.volumes))
 
     @mock.patch('zun.common.utils.spawn_n')
@@ -402,9 +414,11 @@ class TestManager(base.TestCase):
                        side_effect=FakeVolumeMapping.list_by_container)
     @mock.patch.object(fake_driver, 'detach_volume')
     @mock.patch.object(fake_driver, 'attach_volume')
+    @mock.patch.object(fake_driver, 'is_volume_available')
     @mock.patch('zun.image.driver.pull_image')
     def test_container_run_image_pull_docker_error(
-            self, mock_pull, mock_attach_volume, mock_detach_volume,
+            self, mock_pull, mock_is_volume_available,
+            mock_attach_volume, mock_detach_volume,
             mock_list_by_container, mock_save, mock_spawn_n):
         container_dict = utils.get_test_container(
             image='test:latest', image_driver='docker',
@@ -428,6 +442,7 @@ class TestManager(base.TestCase):
                                           'ifnotpresent', 'docker')
         mock_attach_volume.assert_called_once()
         mock_detach_volume.assert_called_once()
+        mock_is_volume_available.assert_called_once()
         self.assertEqual(0, len(FakeVolumeMapping.volumes))
 
     @mock.patch('zun.common.utils.spawn_n')
@@ -436,12 +451,13 @@ class TestManager(base.TestCase):
                        side_effect=FakeVolumeMapping.list_by_container)
     @mock.patch.object(fake_driver, 'detach_volume')
     @mock.patch.object(fake_driver, 'attach_volume')
+    @mock.patch.object(fake_driver, 'is_volume_available')
     @mock.patch('zun.image.driver.pull_image')
     @mock.patch.object(fake_driver, 'create')
     def test_container_run_create_raises_docker_error(
-            self, mock_create, mock_pull, mock_attach_volume,
+            self, mock_create, mock_pull, mock_is_volume_available,
-            mock_detach_volume, mock_list_by_container, mock_save,
+            mock_attach_volume, mock_detach_volume,
-            mock_spawn_n):
+            mock_list_by_container, mock_save, mock_spawn_n):
         container = Container(self.context, **utils.get_test_container())
         image = {'image': 'repo', 'path': 'out_path', 'driver': 'glance',
                  'repo': 'test', 'tag': 'testtag'}
@@ -467,6 +483,7 @@ class TestManager(base.TestCase):
             self.context, container, image, networks, volumes)
         mock_attach_volume.assert_called_once()
         mock_detach_volume.assert_called_once()
+        mock_is_volume_available.assert_called_once()
         self.assertEqual(0, len(FakeVolumeMapping.volumes))
 
     @mock.patch.object(FakeResourceTracker,

zun/volume/driver.py

@@ -23,9 +23,9 @@ from zun.common import exception
 from zun.common.i18n import _
 from zun.common import mount
 import zun.conf
+from zun.volume import cinder_api
 from zun.volume import cinder_workflow
 
-
 LOG = logging.getLogger(__name__)
 
 CONF = zun.conf.CONF
@@ -72,6 +72,9 @@ class VolumeDriver(object):
     def bind_mount(self, *args, **kwargs):
         raise NotImplementedError()
 
+    def is_volume_available(self, *args, **kwargs):
+        raise NotImplementedError()
+
 
 class Cinder(VolumeDriver):
 
@@ -116,3 +119,10 @@ class Cinder(VolumeDriver):
     def bind_mount(self, volume):
         mountpoint = mount.get_mountpoint(volume.volume_id)
         return mountpoint, volume.container_path
+
+    def is_volume_available(self, volume):
+        ca = cinder_api.CinderAPI(self.context)
+        if 'available' == ca.get(volume.volume_id).status:
+            return True
+        else:
+            return False