From bf3e51e5b24f04d792f37d48deb09c268e7b442f Mon Sep 17 00:00:00 2001 From: Simon Dodsley Date: Tue, 30 Aug 2022 11:00:09 -0400 Subject: [PATCH] [Pure Storage] Add replication support for NVMe driver Update driver with replication support prior to initial release of this driver in Zed. Change-Id: I75be679ec3cabe0d93534ec3f0115875295db630 --- cinder/tests/unit/volume/drivers/test_pure.py | 241 +++++++++++++++--- cinder/volume/drivers/pure.py | 7 - .../drivers/pure-storage-driver.rst | 9 +- ...-storage-nvme-driver-f4217c00379c4827.yaml | 2 +- 4 files changed, 211 insertions(+), 48 deletions(-) diff --git a/cinder/tests/unit/volume/drivers/test_pure.py b/cinder/tests/unit/volume/drivers/test_pure.py index d6909d112ba..8a7e787f3fd 100644 --- a/cinder/tests/unit/volume/drivers/test_pure.py +++ b/cinder/tests/unit/volume/drivers/test_pure.py @@ -87,6 +87,10 @@ FC_PORT_NAMES = ["ct0.fc2", "ct0.fc3", "ct1.fc2", "ct1.fc3"] NVME_IPS = ["10.0.0." + str(i + 1) for i in range(len(NVME_PORT_NAMES))] NVME_IPS += ["[2001:db8::" + str(i + 1) + "]" for i in range(len(NVME_PORT_NAMES))] +AC_NVME_IPS = ["10.0.0." + str(i + 1 + len(NVME_PORT_NAMES)) + for i in range(len(NVME_PORT_NAMES))] +AC_NVME_IPS += ["[2001:db8::1:" + str(i + 1) + "]" + for i in range(len(NVME_PORT_NAMES))] NVME_CIDR = "0.0.0.0/0" NVME_CIDR_V6 = "::/0" NVME_PORT = 4420 @@ -131,6 +135,7 @@ NVME_CONNECTOR = {"nqn": INITIATOR_NQN, "host": HOSTNAME} ISCSI_CONNECTOR = {"initiator": INITIATOR_IQN, "host": HOSTNAME} FC_CONNECTOR = {"wwpns": {INITIATOR_WWN}, "host": HOSTNAME} TARGET_NQN = "nqn.2010-06.com.purestorage:flasharray.12345abc" +AC_TARGET_NQN = "nqn.2010-06.com.purestorage:flasharray.67890def" TARGET_IQN = "iqn.2010-06.com.purestorage:flasharray.12345abc" AC_TARGET_IQN = "iqn.2018-06.com.purestorage:flasharray.67890def" TARGET_WWN = "21000024ff59fe94" @@ -166,6 +171,12 @@ NVME_PORTS = [{"name": name, "portal": ip + ":" + TARGET_ROCE_PORT, "wwn": None, } for name, ip in zip(NVME_PORT_NAMES * 2, NVME_IPS)] +AC_NVME_PORTS = [{"name": name, + "nqn": AC_TARGET_NQN, + "iqn": None, + "portal": ip + ":" + TARGET_ROCE_PORT, + "wwn": None, + } for name, ip in zip(NVME_PORT_NAMES * 2, AC_NVME_IPS)] ISCSI_PORTS = [{"name": name, "iqn": TARGET_IQN, "portal": ip + ":" + TARGET_PORT, @@ -340,7 +351,55 @@ NVME_CONNECTION_INFO_V6 = { "volume_nguid": "0009714b5cb916324a9374c470002b2c8", }, } - +NVME_CONNECTION_INFO_AC = { + "driver_volume_type": "nvmeof", + "data": { + "target_nqn": TARGET_NQN, + "discard": True, + "portals": [ + (NVME_IPS[0], NVME_PORT, "rdma"), + (NVME_IPS[1], NVME_PORT, "rdma"), + (NVME_IPS[2], NVME_PORT, "rdma"), + (NVME_IPS[3], NVME_PORT, "rdma"), + (AC_NVME_IPS[0], NVME_PORT, "rdma"), + (AC_NVME_IPS[1], NVME_PORT, "rdma"), + (AC_NVME_IPS[2], NVME_PORT, "rdma"), + (AC_NVME_IPS[3], NVME_PORT, "rdma")], + "volume_nguid": "0009714b5cb916324a9374c470002b2c8", + }, +} +NVME_CONNECTION_INFO_AC_FILTERED = { + "driver_volume_type": "nvmeof", + "data": { + "target_nqn": TARGET_NQN, + "discard": True, + # Final entry filtered by NVME_CIDR_FILTERED + "portals": [ + (NVME_IPS[0], NVME_PORT, "rdma"), + (NVME_IPS[1], NVME_PORT, "rdma"), + (NVME_IPS[2], NVME_PORT, "rdma"), + (NVME_IPS[3], NVME_PORT, "rdma"), + (AC_NVME_IPS[0], NVME_PORT, "rdma"), + (AC_NVME_IPS[1], NVME_PORT, "rdma"), + (AC_NVME_IPS[2], NVME_PORT, "rdma")], + "volume_nguid": "0009714b5cb916324a9374c470002b2c8", + }, +} +NVME_CONNECTION_INFO_AC_FILTERED_LIST = { + "driver_volume_type": "nvmeof", + "data": { + "target_nqn": TARGET_NQN, + "discard": True, + # Final entry filtered by NVME_CIDR_FILTERED + "portals": [ + (NVME_IPS[1], NVME_PORT, "rdma"), + (NVME_IPS[2], NVME_PORT, "rdma"), + (AC_NVME_IPS[5].strip("[]"), NVME_PORT, "rdma"), # IPv6 + (AC_NVME_IPS[6].strip("[]"), NVME_PORT, "rdma"), # IPv6 + ], + "volume_nguid": "0009714b5cb916324a9374c470002b2c8", + }, +} FC_CONNECTION_INFO = { "driver_volume_type": "fibre_channel", "data": { @@ -978,37 +1037,6 @@ class PureBaseVolumeDriverTestCase(PureBaseSharedDriverTestCase): mock.call(self.array, [mock_sync_target], 'cinder-pod') ]) - @mock.patch(BASE_DRIVER_OBJ + '._setup_replicated_pods') - @mock.patch(BASE_DRIVER_OBJ + '._generate_replication_retention') - @mock.patch(BASE_DRIVER_OBJ + '._setup_replicated_pgroups') - def test_do_setup_replicated_sync_rep_bad_driver( - self, - mock_setup_repl_pgroups, - mock_generate_replication_retention, - mock_setup_pods): - retention = mock.MagicMock() - mock_generate_replication_retention.return_value = retention - self._setup_mocks_for_replication() - - self.mock_config.safe_get.return_value = [ - { - "backend_id": "foo", - "managed_backend_name": None, - "san_ip": "1.2.3.4", - "api_token": "abc123", - "type": "sync", - } - ] - mock_sync_target = mock.MagicMock() - mock_sync_target.get.return_value = GET_ARRAY_SECONDARY - self.array.get.return_value = GET_ARRAY_PRIMARY - self.driver._storage_protocol = 'NVMe-RoCE' - self.purestorage_module.FlashArray.side_effect = [self.array, - mock_sync_target] - self.assertRaises(pure.PureDriverException, - self.driver.do_setup, - None) - def test_update_provider_info_update_all(self): test_vols = [ self.new_fake_vol(spec={'id': fake.VOLUME_ID}, @@ -4520,6 +4548,155 @@ class PureNVMEDriverTestCase(PureBaseSharedDriverTestCase): NVME_CONNECTOR, ) + @mock.patch(NVME_DRIVER_OBJ + "._get_nguid") + @mock.patch(NVME_DRIVER_OBJ + "._get_wwn") + @mock.patch(NVME_DRIVER_OBJ + "._connect") + @mock.patch(NVME_DRIVER_OBJ + "._get_target_nvme_ports") + def test_initialize_connection_uniform_ac( + self, mock_get_nvme_ports, mock_connection, mock_get_wwn, + mock_get_nguid + ): + repl_extra_specs = { + "replication_type": " sync", + "replication_enabled": " true", + } + vol, vol_name = self.new_fake_vol(type_extra_specs=repl_extra_specs) + mock_get_nvme_ports.side_effect = [NVME_PORTS, AC_NVME_PORTS] + mock_get_wwn.return_value = "3624a93709714b5cb91634c470002b2c8" + mock_get_nguid.return_value = "0009714b5cb916324a9374c470002b2c8" + mock_connection.side_effect = [ + { + "vol": vol_name, + "lun": 1, + }, + { + "vol": vol_name, + "lun": 5, + }, + ] + result = deepcopy(NVME_CONNECTION_INFO_AC) + + self.driver._is_active_cluster_enabled = True + mock_secondary = mock.MagicMock() + self.driver._uniform_active_cluster_target_arrays = [mock_secondary] + + real_result = self.driver.initialize_connection(vol, NVME_CONNECTOR) + self.assertDictEqual(result, real_result) + mock_get_nvme_ports.assert_has_calls( + [ + mock.call(self.array), + mock.call(mock_secondary), + ] + ) + mock_connection.assert_has_calls( + [ + mock.call(self.array, vol_name, NVME_CONNECTOR), + mock.call( + mock_secondary, vol_name, NVME_CONNECTOR), + ] + ) + + @mock.patch(NVME_DRIVER_OBJ + "._get_nguid") + @mock.patch(NVME_DRIVER_OBJ + "._get_wwn") + @mock.patch(NVME_DRIVER_OBJ + "._connect") + @mock.patch(NVME_DRIVER_OBJ + "._get_target_nvme_ports") + def test_initialize_connection_uniform_ac_cidr( + self, mock_get_nvme_ports, mock_connection, mock_get_wwn, + mock_get_nguid + ): + repl_extra_specs = { + "replication_type": " sync", + "replication_enabled": " true", + } + vol, vol_name = self.new_fake_vol(type_extra_specs=repl_extra_specs) + mock_get_nvme_ports.side_effect = [NVME_PORTS, AC_NVME_PORTS] + mock_get_wwn.return_value = "3624a93709714b5cb91634c470002b2c8" + mock_get_nguid.return_value = "0009714b5cb916324a9374c470002b2c8" + mock_connection.side_effect = [ + { + "vol": vol_name, + "lun": 1, + }, + { + "vol": vol_name, + "lun": 5, + }, + ] + result = deepcopy(NVME_CONNECTION_INFO_AC_FILTERED) + self.driver._is_active_cluster_enabled = True + # Set up some CIDRs to block: this will block only one of the + # get four+three results back + self.driver.configuration.pure_nvme_cidr = NVME_CIDR_FILTERED + mock_secondary = mock.MagicMock() + self.driver._uniform_active_cluster_target_arrays = [mock_secondary] + + real_result = self.driver.initialize_connection(vol, NVME_CONNECTOR) + self.assertDictEqual(result, real_result) + mock_get_nvme_ports.assert_has_calls( + [ + mock.call(self.array), + mock.call(mock_secondary), + ] + ) + mock_connection.assert_has_calls( + [ + mock.call(self.array, vol_name, NVME_CONNECTOR), + mock.call(mock_secondary, vol_name, NVME_CONNECTOR), + ] + ) + + @mock.patch(NVME_DRIVER_OBJ + "._get_nguid") + @mock.patch(NVME_DRIVER_OBJ + "._get_wwn") + @mock.patch(NVME_DRIVER_OBJ + "._connect") + @mock.patch(NVME_DRIVER_OBJ + "._get_target_nvme_ports") + def test_initialize_connection_uniform_ac_cidrs( + self, mock_get_nvme_ports, mock_connection, mock_get_wwn, + mock_get_nguid + ): + repl_extra_specs = { + "replication_type": " sync", + "replication_enabled": " true", + } + vol, vol_name = self.new_fake_vol(type_extra_specs=repl_extra_specs) + mock_get_nvme_ports.side_effect = [NVME_PORTS, AC_NVME_PORTS] + mock_get_wwn.return_value = "3624a93709714b5cb91634c470002b2c8" + mock_get_nguid.return_value = "0009714b5cb916324a9374c470002b2c8" + mock_connection.side_effect = [ + { + "vol": vol_name, + "lun": 1, + }, + { + "vol": vol_name, + "lun": 5, + }, + ] + result = deepcopy(NVME_CONNECTION_INFO_AC_FILTERED_LIST) + + self.driver._is_active_cluster_enabled = True + # Set up some CIDRs to block: this will allow only 2 addresses from + # each host of the ActiveCluster, so we should check that we only + # get two+two results back + self.driver.configuration.pure_nvme = NVME_CIDR + self.driver.configuration.pure_nvme_cidr_list = NVME_CIDRS_FILTERED + mock_secondary = mock.MagicMock() + self.driver._uniform_active_cluster_target_arrays = [mock_secondary] + + real_result = self.driver.initialize_connection(vol, NVME_CONNECTOR) + self.assertDictEqual(result, real_result) + mock_get_nvme_ports.assert_has_calls( + [ + mock.call(self.array), + mock.call(mock_secondary), + ] + ) + mock_connection.assert_has_calls( + [ + mock.call(self.array, vol_name, NVME_CONNECTOR), + mock.call(mock_secondary, vol_name, NVME_CONNECTOR), + ] + ) + @mock.patch(NVME_DRIVER_OBJ + "._get_nguid") @mock.patch(NVME_DRIVER_OBJ + "._get_wwn") @mock.patch(NVME_DRIVER_OBJ + "._connect") diff --git a/cinder/volume/drivers/pure.py b/cinder/volume/drivers/pure.py index ebeb59eaa10..4829755a0ce 100644 --- a/cinder/volume/drivers/pure.py +++ b/cinder/volume/drivers/pure.py @@ -282,13 +282,6 @@ class PureBaseVolumeDriver(san.SanDriver): ssl_cert_path = replication_device.get("ssl_cert_path", None) repl_type = replication_device.get("type", REPLICATION_TYPE_ASYNC) - if ( - repl_type == REPLICATION_TYPE_SYNC - and "NVMe" in self._storage_protocol - ): - msg = _('NVMe driver does not support synchronous ' - 'replication') - raise PureDriverException(reason=msg) uniform = strutils.bool_from_string( replication_device.get("uniform", False)) diff --git a/doc/source/configuration/block-storage/drivers/pure-storage-driver.rst b/doc/source/configuration/block-storage/drivers/pure-storage-driver.rst index 930ced4c0f5..56cf10fe2e4 100644 --- a/doc/source/configuration/block-storage/drivers/pure-storage-driver.rst +++ b/doc/source/configuration/block-storage/drivers/pure-storage-driver.rst @@ -26,8 +26,6 @@ means you do not have the high-availability and non-disruptive upgrade benefits provided by FlashArray. Multipathing must be used to take advantage of these benefits. -The NVMe driver does not support synchronous replication using ActiveCluster. - Supported operations ~~~~~~~~~~~~~~~~~~~~ @@ -51,8 +49,7 @@ Supported operations * Create a thin provisioned volume. -* Replicate volumes to remote Pure Storage array(s) - synchronous replication - is not supported with the NVMe driver. +* Replicate volumes to remote Pure Storage array(s) QoS support for the Pure Storage drivers include the ability to set the following capabilities in the OpenStack Block Storage API @@ -267,10 +264,6 @@ of the remote array. The ``REPLICATION_TYPE`` value for the ``type`` key can be either ``sync`` or ``async`` -.. note:: - - Synchronous replication is not supported by the NVMe driver. - If the ``type`` is ``sync`` volumes will be created in a stretched Pod. This requires two arrays pre-configured with Active Cluster enabled. You can optionally specify ``uniform`` as ``true`` or ``false``, this will instruct diff --git a/releasenotes/notes/pure-storage-nvme-driver-f4217c00379c4827.yaml b/releasenotes/notes/pure-storage-nvme-driver-f4217c00379c4827.yaml index c72121465ff..3fd81a5267c 100644 --- a/releasenotes/notes/pure-storage-nvme-driver-f4217c00379c4827.yaml +++ b/releasenotes/notes/pure-storage-nvme-driver-f4217c00379c4827.yaml @@ -4,4 +4,4 @@ features: Pure Storage adds a new driver to support NVMe-RoCE for the FlashArray. All features of the iSCSI and FC drivers are fully supported by this new - driver with the exception of synchronous replication. + driver.