Fix broken gate jobs

This patch aims at fixing the broken gate jobs because of
cinder and glance patches [1], [2], [3] and [4].

* Remove parameter `--source-replicated` to drop volume
  replication v1 support
* Address some timing issues with volume transfer requests
* Only run Image v1 tests when the test cloud has v1 available
* Get tolerant of unexpected additional attributes being
  returned in Image data

[1].https://review.openstack.org/#/c/586293/
[2].https://review.openstack.org/#/c/532503/
[3].https://review.openstack.org/#/c/533564/
[4].https://review.openstack.org/#/c/578755/

Co-Authored-By: Dean Troyer <dtroyer@gmail.com>
Co-Authored-By: Monty Taylor <mordred@inaugust.com>
Depends-on: https://review.openstack.org/588664
Change-Id: I2a785750e92155185d3344e6116c7f5c6fdd3cbe
Signed-off-by: Fan Zhang <zh.f@outlook.com>

doc/source/cli/command-objects/volume.rst

@@ -15,7 +15,7 @@ Create new volume
     openstack volume create
         [--size <size>]
         [--type <volume-type>]
-        [--image <image> | --snapshot <snapshot> | --source <volume> | --source-replicated <replicated-volume>]
+        [--image <image> | --snapshot <snapshot> | --source <volume> ]
         [--description <description>]
         [--user <user>]
         [--project <project>]
@@ -31,7 +31,7 @@ Create new volume
 .. option:: --size <size>
 
     Volume size in GB
-    (Required unless --snapshot or --source or --source-replicated is specified)
+    (Required unless --snapshot or --source is specified)
 
 .. option:: --type <volume-type>
 
@@ -54,10 +54,6 @@ Create new volume
 
     Volume to clone (name or ID)
 
-.. option:: --source-replicated <replicated-volume>
-
-    Replicated volume to clone (name or ID)
-
 .. option:: --description <description>
 
     Volume description

openstackclient/tests/functional/image/base.py

@@ -0,0 +1,24 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from openstackclient.tests.functional import base
+
+
+class BaseImageTests(base.TestCase):
+    """Functional tests for Image commands"""
+
+    @classmethod
+    def setUpClass(cls):
+        super(BaseImageTests, cls).setUpClass()
+        # TODO(dtroyer): maybe do image API discovery here to determine
+        #                what is available, it isn't in the service catalog
+        cls.haz_v1_api = False

openstackclient/tests/functional/image/v1/test_image.py

@@ -15,50 +15,47 @@ import uuid
 
 import fixtures
 
-from openstackclient.tests.functional import base
+from openstackclient.tests.functional.image import base
 
 
-class ImageTests(base.TestCase):
+class ImageTests(base.BaseImageTests):
-    """Functional tests for image. """
+    """Functional tests for Image commands"""
-
-    NAME = uuid.uuid4().hex
-    OTHER_NAME = uuid.uuid4().hex
-
-    @classmethod
-    def setUpClass(cls):
-        super(ImageTests, cls).setUpClass()
-        json_output = json.loads(cls.openstack(
-            '--os-image-api-version 1 '
-            'image create -f json ' +
-            cls.NAME
-        ))
-        cls.image_id = json_output["id"]
-        cls.assertOutput(cls.NAME, json_output['name'])
-
-    @classmethod
-    def tearDownClass(cls):
-        try:
-            cls.openstack(
-                '--os-image-api-version 1 '
-                'image delete ' +
-                cls.image_id
-            )
-        finally:
-            super(ImageTests, cls).tearDownClass()
 
     def setUp(self):
         super(ImageTests, self).setUp()
+        if not self.haz_v1_api:
+            self.skipTest('No Image v1 API present')
+
+        self.name = uuid.uuid4().hex
+        json_output = json.loads(self.openstack(
+            '--os-image-api-version 1 '
+            'image create -f json ' +
+            self.name
+        ))
+        self.image_id = json_output["id"]
+        self.assertOutput(self.name, json_output['name'])
+
         ver_fixture = fixtures.EnvironmentVariable(
             'OS_IMAGE_API_VERSION', '1'
         )
         self.useFixture(ver_fixture)
 
+    def tearDown(self):
+        try:
+            self.openstack(
+                '--os-image-api-version 1 '
+                'image delete ' +
+                self.image_id
+            )
+        finally:
+            super(ImageTests, self).tearDown()
+
     def test_image_list(self):
         json_output = json.loads(self.openstack(
             'image list -f json '
         ))
         self.assertIn(
-            self.NAME,
+            self.name,
             [img['Name'] for img in json_output]
         )
 
@@ -72,11 +69,11 @@ class ImageTests(base.TestCase):
             '--min-ram 5 ' +
             '--disk-format qcow2 ' +
             '--public ' +
-            self.NAME
+            self.name
         )
         json_output = json.loads(self.openstack(
             'image show -f json ' +
-            self.NAME
+            self.name
         ))
         self.assertEqual(
             4,
@@ -100,11 +97,11 @@ class ImageTests(base.TestCase):
             '--property a=b ' +
             '--property c=d ' +
             '--public ' +
-            self.NAME
+            self.name
         )
         json_output = json.loads(self.openstack(
             'image show -f json ' +
-            self.NAME
+            self.name
         ))
         self.assertEqual(
             "a='b', c='d'",

openstackclient/tests/functional/image/v2/test_image.py

@@ -16,60 +16,55 @@ import uuid
 import fixtures
 # from glanceclient import exc as image_exceptions
 
-from openstackclient.tests.functional import base
+from openstackclient.tests.functional.image import base
 
 
-class ImageTests(base.TestCase):
+class ImageTests(base.BaseImageTests):
-    """Functional tests for image. """
+    """Functional tests for Image commands"""
-
-    NAME = uuid.uuid4().hex
-    OTHER_NAME = uuid.uuid4().hex
-
-    @classmethod
-    def setUpClass(cls):
-        super(ImageTests, cls).setUpClass()
-        cls.image_tag = 'my_tag'
-        json_output = json.loads(cls.openstack(
-            '--os-image-api-version 2 '
-            'image create -f json --tag {tag} {name}'.format(
-                tag=cls.image_tag, name=cls.NAME)
-        ))
-        cls.image_id = json_output["id"]
-        cls.assertOutput(cls.NAME, json_output['name'])
-
-    @classmethod
-    def tearDownClass(cls):
-        try:
-            cls.openstack(
-                '--os-image-api-version 2 '
-                'image delete ' +
-                cls.image_id
-            )
-        finally:
-            super(ImageTests, cls).tearDownClass()
 
     def setUp(self):
         super(ImageTests, self).setUp()
+
+        self.name = uuid.uuid4().hex
+        self.image_tag = 'my_tag'
+        json_output = json.loads(self.openstack(
+            '--os-image-api-version 2 '
+            'image create -f json --tag {tag} {name}'.format(
+                tag=self.image_tag, name=self.name)
+        ))
+        self.image_id = json_output["id"]
+        self.assertOutput(self.name, json_output['name'])
+
         ver_fixture = fixtures.EnvironmentVariable(
             'OS_IMAGE_API_VERSION', '2'
         )
         self.useFixture(ver_fixture)
 
+    def tearDown(self):
+        try:
+            self.openstack(
+                '--os-image-api-version 2 '
+                'image delete ' +
+                self.image_id
+            )
+        finally:
+            super(ImageTests, self).tearDown()
+
     def test_image_list(self):
         json_output = json.loads(self.openstack(
             'image list -f json '
         ))
         self.assertIn(
-            self.NAME,
+            self.name,
             [img['Name'] for img in json_output]
         )
 
     def test_image_list_with_name_filter(self):
         json_output = json.loads(self.openstack(
-            'image list --name ' + self.NAME + ' -f json'
+            'image list --name ' + self.name + ' -f json'
         ))
         self.assertIn(
-            self.NAME,
+            self.name,
             [img['Name'] for img in json_output]
         )
 
@@ -101,11 +96,11 @@ class ImageTests(base.TestCase):
             '--min-disk 4 ' +
             '--min-ram 5 ' +
             '--public ' +
-            self.NAME
+            self.name
         )
         json_output = json.loads(self.openstack(
             'image show -f json ' +
-            self.NAME
+            self.name
         ))
         self.assertEqual(
             4,
@@ -126,31 +121,31 @@ class ImageTests(base.TestCase):
             '--property a=b ' +
             '--property c=d ' +
             '--public ' +
-            self.NAME
+            self.name
         )
         json_output = json.loads(self.openstack(
             'image show -f json ' +
-            self.NAME
+            self.name
         ))
-        self.assertEqual(
+        # NOTE(dtroyer): Don't do a full-string compare so we are tolerant of
-            "a='b', c='d'",
+        #                new artributes in the returned data
-            json_output["properties"],
+        self.assertIn("a='b'", json_output["properties"])
-        )
+        self.assertIn("c='d'", json_output["properties"])
 
         self.openstack(
             'image unset ' +
             '--property a ' +
             '--property c ' +
-            self.NAME
+            self.name
         )
         json_output = json.loads(self.openstack(
             'image show -f json ' +
-            self.NAME
+            self.name
         ))
-        self.assertNotIn(
+        # NOTE(dtroyer): Don't do a full-string compare so we are tolerant of
-            'properties',
+        #                new artributes in the returned data
-            json_output,
+        self.assertNotIn("a='b'", json_output["properties"])
-        )
+        self.assertNotIn("c='d'", json_output["properties"])
 
         # Test tags
         self.assertNotIn(
@@ -160,11 +155,11 @@ class ImageTests(base.TestCase):
         self.openstack(
             'image set ' +
             '--tag 01 ' +
-            self.NAME
+            self.name
         )
         json_output = json.loads(self.openstack(
             'image show -f json ' +
-            self.NAME
+            self.name
         ))
         self.assertIn(
             '01',
@@ -174,11 +169,11 @@ class ImageTests(base.TestCase):
         self.openstack(
             'image unset ' +
             '--tag 01 ' +
-            self.NAME
+            self.name
         )
         json_output = json.loads(self.openstack(
             'image show -f json ' +
-            self.NAME
+            self.name
         ))
         self.assertNotIn(
             '01',
@@ -222,7 +217,7 @@ class ImageTests(base.TestCase):
 
         json_output = json.loads(self.openstack(
             'image show -f json ' +
-            self.NAME
+            self.name
         ))
         # NOTE(dtroyer): Until OSC supports --shared flags in create and set
         #                we can not properly test membership.  Sometimes the
@@ -230,47 +225,47 @@ class ImageTests(base.TestCase):
         if json_output["visibility"] == 'shared':
             self.openstack(
                 'image add project ' +
-                self.NAME + ' ' +
+                self.name + ' ' +
                 my_project_id
             )
             # self.addCleanup(
             #     self.openstack,
             #     'image remove project ' +
-            #     self.NAME + ' ' +
+            #     self.name + ' ' +
             #     my_project_id
             # )
 
             self.openstack(
                 'image set ' +
                 '--accept ' +
-                self.NAME
+                self.name
             )
             json_output = json.loads(self.openstack(
                 'image list -f json ' +
                 '--shared'
             ))
             self.assertIn(
-                self.NAME,
+                self.name,
                 [img['Name'] for img in json_output]
             )
 
             self.openstack(
                 'image set ' +
                 '--reject ' +
-                self.NAME
+                self.name
             )
             json_output = json.loads(self.openstack(
                 'image list -f json ' +
                 '--shared'
             ))
             # self.assertNotIn(
-            #     self.NAME,
+            #     self.name,
             #     [img['Name'] for img in json_output]
             # )
 
             self.openstack(
                 'image remove project ' +
-                self.NAME + ' ' +
+                self.name + ' ' +
                 my_project_id
             )
 
@@ -280,11 +275,11 @@ class ImageTests(base.TestCase):
         #         image_exceptions.HTTPForbidden,
         #         self.openstack,
         #         'image add project ' +
-        #         self.NAME + ' ' +
+        #         self.name + ' ' +
         #         my_project_id
         #     )
         #     self.openstack(
         #         'image set ' +
         #         '--share ' +
-        #         self.NAME
+        #         self.name
         #     )

openstackclient/tests/functional/volume/v1/test_transfer_request.py

@@ -29,20 +29,13 @@ class TransferRequestTests(common.BaseVolumeTests):
             'volume create -f json --size 1 ' + cls.VOLUME_NAME))
         cls.assertOutput(cls.VOLUME_NAME, cmd_output['name'])
 
-        cmd_output = json.loads(cls.openstack(
+        cls.wait_for_status("volume", cls.VOLUME_NAME, "available")
-            'volume transfer request create -f json ' +
-            cls.VOLUME_NAME +
-            ' --name ' + cls.NAME))
-        cls.assertOutput(cls.NAME, cmd_output['name'])
 
     @classmethod
     def tearDownClass(cls):
         try:
-            raw_output_transfer = cls.openstack(
-                'volume transfer request delete ' + cls.NAME)
             raw_output_volume = cls.openstack(
                 'volume delete ' + cls.VOLUME_NAME)
-            cls.assertOutput('', raw_output_transfer)
             cls.assertOutput('', raw_output_volume)
         finally:
             super(TransferRequestTests, cls).tearDownClass()
@@ -79,12 +72,28 @@ class TransferRequestTests(common.BaseVolumeTests):
             'volume delete ' + volume_name)
         self.assertEqual('', raw_output)
 
-    def test_volume_transfer_request_list(self):
+    def test_volume_transfer_request_list_show(self):
+        name = uuid.uuid4().hex
         cmd_output = json.loads(self.openstack(
-            'volume transfer request list -f json'))
+            'volume transfer request create -f json ' +
-        self.assertIn(self.NAME, [req['Name'] for req in cmd_output])
+            ' --name ' + name + ' ' +
+            self.VOLUME_NAME
+        ))
+        self.addCleanup(
+            self.openstack,
+            'volume transfer request delete ' + name
+        )
+        self.assertOutput(name, cmd_output['name'])
+        auth_key = cmd_output['auth_key']
+        self.assertTrue(auth_key)
 
-    def test_volume_transfer_request_show(self):
         cmd_output = json.loads(self.openstack(
-            'volume transfer request show -f json ' + self.NAME))
+            'volume transfer request list -f json'
-        self.assertEqual(self.NAME, cmd_output['name'])
+        ))
+        self.assertIn(name, [req['Name'] for req in cmd_output])
+
+        cmd_output = json.loads(self.openstack(
+            'volume transfer request show -f json ' +
+            name
+        ))
+        self.assertEqual(name, cmd_output['name'])

openstackclient/tests/functional/volume/v2/test_transfer_request.py

@@ -21,29 +21,24 @@ class TransferRequestTests(common.BaseVolumeTests):
 
     NAME = uuid.uuid4().hex
     VOLUME_NAME = uuid.uuid4().hex
+    API_VERSION = '2'
 
     @classmethod
     def setUpClass(cls):
         super(TransferRequestTests, cls).setUpClass()
 
         cmd_output = json.loads(cls.openstack(
+            '--os-volume-api-version ' + cls.API_VERSION + ' ' +
             'volume create -f json --size 1 ' + cls.VOLUME_NAME))
         cls.assertOutput(cls.VOLUME_NAME, cmd_output['name'])
 
-        cmd_output = json.loads(cls.openstack(
+        cls.wait_for_status("volume", cls.VOLUME_NAME, "available")
-            'volume transfer request create -f json ' +
-            cls.VOLUME_NAME +
-            ' --name ' + cls.NAME))
-        cls.assertOutput(cls.NAME, cmd_output['name'])
 
     @classmethod
     def tearDownClass(cls):
         try:
-            raw_output_transfer = cls.openstack(
-                'volume transfer request delete ' + cls.NAME)
             raw_output_volume = cls.openstack(
                 'volume delete ' + cls.VOLUME_NAME)
-            cls.assertOutput('', raw_output_transfer)
             cls.assertOutput('', raw_output_volume)
         finally:
             super(TransferRequestTests, cls).tearDownClass()
@@ -80,12 +75,31 @@ class TransferRequestTests(common.BaseVolumeTests):
             'volume delete ' + volume_name)
         self.assertEqual('', raw_output)
 
-    def test_volume_transfer_request_list(self):
+    def test_volume_transfer_request_list_show(self):
+        name = uuid.uuid4().hex
         cmd_output = json.loads(self.openstack(
-            'volume transfer request list -f json'))
+            '--os-volume-api-version ' + self.API_VERSION + ' ' +
-        self.assertIn(self.NAME, [req['Name'] for req in cmd_output])
+            'volume transfer request create -f json ' +
+            ' --name ' + name + ' ' +
+            self.VOLUME_NAME
+        ))
+        self.addCleanup(
+            self.openstack,
+            'volume transfer request delete ' + name
+        )
+        self.assertEqual(name, cmd_output['name'])
+        auth_key = cmd_output['auth_key']
+        self.assertTrue(auth_key)
 
-    def test_volume_transfer_request_show(self):
         cmd_output = json.loads(self.openstack(
-            'volume transfer request show -f json ' + self.NAME))
+            '--os-volume-api-version ' + self.API_VERSION + ' ' +
-        self.assertEqual(self.NAME, cmd_output['name'])
+            'volume transfer request list -f json'
+        ))
+        self.assertIn(name, [req['Name'] for req in cmd_output])
+
+        cmd_output = json.loads(self.openstack(
+            '--os-volume-api-version ' + self.API_VERSION + ' ' +
+            'volume transfer request show -f json ' +
+            name
+        ))
+        self.assertEqual(name, cmd_output['name'])

openstackclient/tests/functional/volume/v3/test_transfer_request.py

@@ -17,3 +17,5 @@ from openstackclient.tests.functional.volume.v3 import common
 
 class TransferRequestTests(common.BaseVolumeTests, v2.TransferRequestTests):
     """Functional tests for transfer request. """
+
+    API_VERSION = '3'

openstackclient/tests/unit/volume/v2/test_volume.py

@@ -133,7 +133,6 @@ class TestVolumeCreate(TestVolume):
             imageRef=None,
             source_volid=None,
             consistencygroup_id=None,
-            source_replica=None,
             multiattach=False,
             scheduler_hints=None,
         )
@@ -185,7 +184,6 @@ class TestVolumeCreate(TestVolume):
             imageRef=None,
             source_volid=None,
             consistencygroup_id=consistency_group.id,
-            source_replica=None,
             multiattach=True,
             scheduler_hints={'k': 'v'},
         )
@@ -231,7 +229,6 @@ class TestVolumeCreate(TestVolume):
             imageRef=None,
             source_volid=None,
             consistencygroup_id=None,
-            source_replica=None,
             multiattach=False,
             scheduler_hints=None,
         )
@@ -277,7 +274,6 @@ class TestVolumeCreate(TestVolume):
             imageRef=None,
             source_volid=None,
             consistencygroup_id=None,
-            source_replica=None,
             multiattach=False,
             scheduler_hints=None,
         )
@@ -317,7 +313,6 @@ class TestVolumeCreate(TestVolume):
             imageRef=None,
             source_volid=None,
             consistencygroup_id=None,
-            source_replica=None,
             multiattach=False,
             scheduler_hints=None,
         )
@@ -359,7 +354,6 @@ class TestVolumeCreate(TestVolume):
             imageRef=image.id,
             source_volid=None,
             consistencygroup_id=None,
-            source_replica=None,
             multiattach=False,
             scheduler_hints=None,
         )
@@ -401,7 +395,6 @@ class TestVolumeCreate(TestVolume):
             imageRef=image.id,
             source_volid=None,
             consistencygroup_id=None,
-            source_replica=None,
             multiattach=False,
             scheduler_hints=None,
         )
@@ -442,7 +435,6 @@ class TestVolumeCreate(TestVolume):
             imageRef=None,
             source_volid=None,
             consistencygroup_id=None,
-            source_replica=None,
             multiattach=False,
             scheduler_hints=None,
         )
@@ -484,7 +476,6 @@ class TestVolumeCreate(TestVolume):
             imageRef=None,
             source_volid=None,
             consistencygroup_id=None,
-            source_replica=None,
             multiattach=False,
             scheduler_hints=None,
         )
@@ -530,7 +521,6 @@ class TestVolumeCreate(TestVolume):
             imageRef=None,
             source_volid=None,
             consistencygroup_id=None,
-            source_replica=None,
             multiattach=False,
             scheduler_hints=None,
         )
@@ -585,7 +575,6 @@ class TestVolumeCreate(TestVolume):
             imageRef=None,
             source_volid=None,
             consistencygroup_id=None,
-            source_replica=None,
             multiattach=False,
             scheduler_hints=None,
         )
@@ -598,40 +587,6 @@ class TestVolumeCreate(TestVolume):
         self.volumes_mock.update_readonly_flag.assert_called_with(
             self.new_volume.id, True)
 
-    def test_volume_create_with_source_replicated(self):
-        self.volumes_mock.get.return_value = self.new_volume
-        arglist = [
-            '--source-replicated', self.new_volume.id,
-            self.new_volume.name,
-        ]
-        verifylist = [
-            ('source_replicated', self.new_volume.id),
-            ('name', self.new_volume.name),
-        ]
-        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
-
-        columns, data = self.cmd.take_action(parsed_args)
-        self.volumes_mock.create.assert_called_once_with(
-            size=None,
-            snapshot_id=None,
-            name=self.new_volume.name,
-            description=None,
-            volume_type=None,
-            user_id=None,
-            project_id=None,
-            availability_zone=None,
-            metadata=None,
-            imageRef=None,
-            source_volid=None,
-            consistencygroup_id=None,
-            source_replica=self.new_volume.id,
-            multiattach=False,
-            scheduler_hints=None,
-        )
-
-        self.assertEqual(self.columns, columns)
-        self.assertEqual(self.datalist, data)
-
     def test_volume_create_without_size(self):
         arglist = [
             self.new_volume.name,
@@ -649,7 +604,6 @@ class TestVolumeCreate(TestVolume):
             '--image', 'source_image',
             '--source', 'source_volume',
             '--snapshot', 'source_snapshot',
-            '--source-replicated', 'source_replicated_volume',
             '--size', str(self.new_volume.size),
             self.new_volume.name,
         ]
@@ -657,7 +611,6 @@ class TestVolumeCreate(TestVolume):
             ('image', 'source_image'),
             ('source', 'source_volume'),
             ('snapshot', 'source_snapshot'),
-            ('source-replicated', 'source_replicated_volume'),
             ('size', self.new_volume.size),
             ('name', self.new_volume.name),
         ]

openstackclient/volume/v2/volume.py

@@ -14,6 +14,7 @@
 
 """Volume V2 Volume action implementations"""
 
+import argparse
 import copy
 import logging
 
@@ -37,7 +38,7 @@ def _check_size_arg(args):
     volume is not specified.
     """
 
-    if ((args.snapshot or args.source or args.source_replicated)
+    if ((args.snapshot or args.source)
             is None and args.size is None):
         msg = _("--size is a required option if snapshot "
                 "or source volume is not specified.")
@@ -59,7 +60,7 @@ class CreateVolume(command.ShowOne):
             metavar="<size>",
             type=int,
             help=_("Volume size in GB (Required unless --snapshot or "
-                   "--source or --source-replicated is specified)"),
+                   "--source is specified)"),
         )
         parser.add_argument(
             "--type",
@@ -85,7 +86,7 @@ class CreateVolume(command.ShowOne):
         source_group.add_argument(
             "--source-replicated",
             metavar="<replicated-volume>",
-            help=_("Replicated volume to clone (name or ID)"),
+            help=argparse.SUPPRESS,
         )
         parser.add_argument(
             "--description",
@@ -168,12 +169,6 @@ class CreateVolume(command.ShowOne):
                 volume_client.volumes,
                 parsed_args.source).id
 
-        replicated_source_volume = None
-        if parsed_args.source_replicated:
-            replicated_source_volume = utils.find_resource(
-                volume_client.volumes,
-                parsed_args.source_replicated).id
-
         consistency_group = None
         if parsed_args.consistency_group:
             consistency_group = utils.find_resource(
@@ -227,7 +222,6 @@ class CreateVolume(command.ShowOne):
             imageRef=image,
             source_volid=source_volume,
             consistencygroup_id=consistency_group,
-            source_replica=replicated_source_volume,
             multiattach=parsed_args.multi_attach,
             scheduler_hints=parsed_args.hint,
         )