diff --git a/cinder/tests/unit/volume/drivers/test_fujitsu_dx.py b/cinder/tests/unit/volume/drivers/test_fujitsu_dx.py index 3fd099d4f35..2ec8fc7310e 100644 --- a/cinder/tests/unit/volume/drivers/test_fujitsu_dx.py +++ b/cinder/tests/unit/volume/drivers/test_fujitsu_dx.py @@ -25,6 +25,8 @@ from cinder import test from cinder.volume import configuration as conf with mock.patch.dict('sys.modules', pywbem=mock.Mock()): + from cinder.volume.drivers.fujitsu.eternus_dx \ + import eternus_dx_cli from cinder.volume.drivers.fujitsu.eternus_dx \ import eternus_dx_common as dx_common from cinder.volume.drivers.fujitsu.eternus_dx \ @@ -888,10 +890,37 @@ class FJFCDriverTestCase(test.TestCase): self.mock_object(dx_common.FJDXCommon, '_create_eternus_instance_name', instancename.fake_create_eternus_instance_name) + self.mock_object(eternus_dx_cli.FJDXCLI, '_exec_cli_with_eternus', + self.fake_exec_cli_with_eternus) # Set iscsi driver to self.driver. driver = dx_fc.FJDXFCDriver(configuration=self.configuration) self.driver = driver + def fake_exec_cli_with_eternus(self, exec_cmdline): + if exec_cmdline == "show users": + ret = ('\r\nCLI> show users\r\n00\r\n' + '3B\r\nf.ce\tMaintainer\t01\t00' + '\t00\t00\r\ntestuser\tSoftware' + '\t01\t01\t00\t00\r\nCLI> ') + return ret + elif exec_cmdline.startswith('show volumes'): + ret = ('\r\nCLI> %s\r\n00\r\n0560\r\n0000' + '\tFJosv_0qJ4rpOHgFE8ipcJOMfBmg==' + '\tA001\t0B\t00\t0000\tabcd1234_TPP' + '\t0000000000200000\t00\t00' + '\t00000000\t0050\tFF\t00\tFF' + '\tFF\t20\tFF\tFFFF\t00' + '\t600000E00D2A0000002A011500140000' + '\t00\t00\tFF\tFF\tFFFFFFFF\t00' + '\t00\tFF\r\n0001\tFJosv_UkCZqMFZW3SU_JzxjHiKfg==' + '\tA001\t0B\t00\t0000\tabcd1234_OSVD' + '\t0000000000200000\t00\t00\t00000000' + '\t0050\tFF\t00\tFF\tFF\t20\tFF\tFFFF' + '\t00\t600000E00D2A0000002A0115001E0000' + '\t00\t00\tFF\tFF\tFFFFFFFF\t00' + '\t00\tFF' % exec_cmdline) + return ret + def fake_safe_get(self, str=None): return str @@ -1029,10 +1058,37 @@ class FJISCSIDriverTestCase(test.TestCase): self.mock_object(dx_common.FJDXCommon, '_get_mapdata_iscsi', self.fake_get_mapdata) + self.mock_object(eternus_dx_cli.FJDXCLI, '_exec_cli_with_eternus', + self.fake_exec_cli_with_eternus) # Set iscsi driver to self.driver. driver = dx_iscsi.FJDXISCSIDriver(configuration=self.configuration) self.driver = driver + def fake_exec_cli_with_eternus(self, exec_cmdline): + if exec_cmdline == "show users": + ret = ('\r\nCLI> show users\r\n00\r\n' + '3B\r\nf.ce\tMaintainer\t01\t00' + '\t00\t00\r\ntestuser\tSoftware' + '\t01\t01\t00\t00\r\nCLI> ') + return ret + elif exec_cmdline.startswith('show volumes'): + ret = ('\r\nCLI> %s\r\n00\r\n0560\r\n0000' + '\tFJosv_0qJ4rpOHgFE8ipcJOMfBmg==' + '\tA001\t0B\t00\t0000\tabcd1234_TPP' + '\t0000000000200000\t00\t00' + '\t00000000\t0050\tFF\t00\tFF' + '\tFF\t20\tFF\tFFFF\t00' + '\t600000E00D2A0000002A011500140000' + '\t00\t00\tFF\tFF\tFFFFFFFF\t00' + '\t00\tFF\r\n0001\tFJosv_UkCZqMFZW3SU_JzxjHiKfg==' + '\tA001\t0B\t00\t0000\tabcd1234_OSVD' + '\t0000000000200000\t00\t00\t00000000' + '\t0050\tFF\t00\tFF\tFF\t20\tFF\tFFFF' + '\t00\t600000E00D2A0000002A0115001E0000' + '\t00\t00\tFF\tFF\tFFFFFFFF\t00' + '\t00\tFF' % exec_cmdline) + return ret + def fake_safe_get(self, str=None): return str diff --git a/cinder/volume/drivers/fujitsu/eternus_dx/constants.py b/cinder/volume/drivers/fujitsu/eternus_dx/constants.py index d8266c6f716..11b05e776d1 100644 --- a/cinder/volume/drivers/fujitsu/eternus_dx/constants.py +++ b/cinder/volume/drivers/fujitsu/eternus_dx/constants.py @@ -24,7 +24,9 @@ BROKEN = 5 JOB_RETRIES = 60 JOB_INTERVAL_SEC = 10 +TIMES_MIN = 3 EC_REC = 3 +RETRY_INTERVAL = 5 # Error code keyword. VOLUME_IS_BUSY = 32786 DEVICE_IS_BUSY = 32787 diff --git a/cinder/volume/drivers/fujitsu/eternus_dx/eternus_dx_cli.py b/cinder/volume/drivers/fujitsu/eternus_dx/eternus_dx_cli.py new file mode 100644 index 00000000000..bd7124b6d8c --- /dev/null +++ b/cinder/volume/drivers/fujitsu/eternus_dx/eternus_dx_cli.py @@ -0,0 +1,264 @@ +# Copyright (c) 2019 FUJITSU LIMITED +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""Cinder Volume driver for Fujitsu ETERNUS DX S3 series.""" +import six + +from cinder.i18n import _ +from cinder import ssh_utils + + +class FJDXCLI(object): + """ETERNUS CLI Code.""" + + def __init__(self, user, storage_ip, password=None, keyfile=None): + """Constructor.""" + self.user = user + self.storage_ip = storage_ip + if password and keyfile: + raise Exception(_('can not specify both password and keyfile')) + + self.use_ipv6 = False + if storage_ip.find(':') != -1: + self.use_ipv6 = True + + if password: + self.ssh_pool = ssh_utils.SSHPool(storage_ip, 22, None, user, + password=password, max_size=2) + + if keyfile: + self.ssh_pool = ssh_utils.SSHPool(storage_ip, 22, None, user, + privatekey=keyfile, max_size=2) + + self.ce_support = False + self.CMD_dic = { + 'check_user_role': self._check_user_role, + 'show_pool_provision': self._show_pool_provision, + } + + self.SMIS_dic = { + '0000': '0', # Success. + '0060': '32787', # The device is in busy state. + '0100': '4097' + } # Size not supported. + + def done(self, command, **option): + func = self.CMD_dic.get(command, self._default_func) + return func(**option) + + def _exec_cli(self, cmd, StrictHostKeyChecking=True, **option): + + exec_cmdline = cmd + self._get_option(**option) + stdoutdata = self._exec_cli_with_eternus(exec_cmdline) + output = [] + message = [] + stdoutlist = stdoutdata.split('\r\n') + output_header = "" + + for no, outline in enumerate(stdoutlist): + if len(outline) <= 0 or outline is None: + continue + + if not output_header.endswith(exec_cmdline): + output_header += outline + continue + + if 0 <= outline.find('Error'): + raise Exception(_("Output: %(outline)s: " + "Command: %(cmdline)s") + % {'outline': outline, + 'cmdline': exec_cmdline}) + + if not self._is_status(outline): + continue + + status = int(outline, 16) + lineno = no + 1 + break + else: + raise Exception(_( + "Invalid CLI output: %(exec_cmdline)s, %(stdoutlist)s") + % {'exec_cmdline': exec_cmdline, + 'stdoutlist': stdoutlist}) + + if status == 0: + rc = '0' + for outline in stdoutlist[lineno:]: + if 0 <= outline.find('CLI>'): + continue + if len(outline) <= 0: + continue + if outline is None: + continue + message.append(outline) + else: + code = stdoutlist[lineno] + for outline in stdoutlist[lineno + 1:]: + if 0 <= outline.find('CLI>'): + continue + if len(outline) <= 0: + continue + if outline is None: + continue + output.append(outline) + + rc, message = self._create_error_message(code, output) + + return {'result': 0, 'rc': rc, 'message': message} + + def _exec_cli_with_eternus(self, exec_cmdline): + """Execute CLI command with arguments.""" + ssh = None + try: + ssh = self.ssh_pool.get() + chan = ssh.invoke_shell() + chan.send(exec_cmdline + '\n') + stdoutdata = '' + while True: + temp = chan.recv(65535) + if isinstance(temp, six.binary_type): + temp = temp.decode('utf-8') + else: + temp = str(temp) + stdoutdata += temp + + # CLI command end with 'CLI>'. + if stdoutdata == '\r\nCLI> ': + continue + if (stdoutdata[len(stdoutdata) - 5: len(stdoutdata) - 1] == + 'CLI>'): + break + except Exception as e: + raise Exception(_("Execute CLI " + "command error. Error: %s") % six.text_type(e)) + finally: + if ssh: + self.ssh_pool.put(ssh) + self.ssh_pool.remove(ssh) + return stdoutdata + + def _create_error_message(self, code, msg): + """Create error code and message using arguements.""" + message = None + if code in self.SMIS_dic: + rc = self.SMIS_dic[code] + else: + rc = 'E' + code + + # TODO(whfnst): we will have a dic to store errors. + if rc == "E0001": + message = "Bad value: %s" % msg + elif rc == "ED184": + message = "Because OPC is being executed, " + "the processing was discontinued." + else: + message = msg + + return rc, message + + @staticmethod + def _is_status(value): + """Check whether input value is status value or not.""" + try: + if len(value) != 2: + return False + + int(value, 16) + int(value[0], 16) + int(value[1], 16) + + return True + except ValueError: + return False + + @staticmethod + def _get_option(**option): + """Create option strings from dictionary.""" + ret = "" + for key, value in six.iteritems(option): + ret += " -%(key)s %(value)s" % {'key': key, 'value': value} + return ret + + def _default_func(self, **option): + """Default function.""" + raise Exception(_("Invalid function is specified")) + + def _check_user_role(self, **option): + """Check user role.""" + try: + output = self._exec_cli("show users", + StrictHostKeyChecking=False, + **option) + # Return error. + rc = output['rc'] + if rc != "0": + return output + + userlist = output.get('message') + role = None + for userinfo in userlist: + username = userinfo.split('\t')[0] + if username == self.user: + role = userinfo.split('\t')[1] + break + + output['message'] = role + except Exception as ex: + if 'show users' in six.text_type(ex): + msg = ("Specified user(%s) does not have Software role" + % self.user) + elif 'Error connecting' in six.text_type(ex): + msg = (six.text_type(ex)[34:] + + ', Please check fujitsu_private_key_path or .xml file') + else: + msg = six.text_type(ex) + output = { + 'result': 0, + 'rc': '4', + 'message': msg + } + return output + + def _show_pool_provision(self, **option): + """Get TPP provision capacity information.""" + try: + output = self._exec_cli("show volumes", **option) + + rc = output['rc'] + + if rc != "0": + return output + + clidatalist = output.get('message') + + data = 0 + for clidataline in clidatalist[1:]: + clidata = clidataline.split('\t') + if clidata[0] == 'FFFF': + break + data += int(clidata[7], 16) + provision = data / 2097152 + + output['message'] = provision + except Exception as ex: + output = { + 'result': 0, + 'rc': '4', + 'message': "show pool provision capacity error: %s" + % six.text_type(ex) + } + + return output diff --git a/cinder/volume/drivers/fujitsu/eternus_dx/eternus_dx_common.py b/cinder/volume/drivers/fujitsu/eternus_dx/eternus_dx_common.py index e9a40c24f8c..8faf203e864 100644 --- a/cinder/volume/drivers/fujitsu/eternus_dx/eternus_dx_common.py +++ b/cinder/volume/drivers/fujitsu/eternus_dx/eternus_dx_common.py @@ -36,6 +36,7 @@ from cinder.i18n import _ from cinder import utils from cinder.volume import configuration as conf from cinder.volume.drivers.fujitsu.eternus_dx import constants as CONSTANTS +from cinder.volume.drivers.fujitsu.eternus_dx import eternus_dx_cli from cinder.volume import volume_utils LOG = logging.getLogger(__name__) @@ -81,6 +82,8 @@ class FJDXCommon(object): self.configuration.iscsi_ip_address = ( self._get_drvcfg('EternusISCSIIP')) self.conn = None + self.fjdxcli = {} + self._check_user() @staticmethod def get_driver_options(): @@ -238,6 +241,8 @@ class FJDXCommon(object): if pool_type == 'RAID': useable_gb = free_gb else: + # If the ratio is less than the value on ETERNUS, + # useable_gb may be negative. Avoid over-allocation. max_capacity = total_gb * float( self.configuration.max_over_subscription_ratio) useable_gb = max_capacity - prov_gb @@ -1158,7 +1163,7 @@ class FJDXCommon(object): target_poolname = list(poolname_list) pools = [] - # Get pools info form CIM instance(include info about instance path). + # Get pools info from CIM instance(include info about instance path). try: tppoollist = self._enum_eternus_instances( 'FUJITSU_ThinProvisioningPool', conn=conn) @@ -1206,6 +1211,28 @@ class FJDXCommon(object): LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) + if ptype == 'TPP': + param_dict = { + 'pool-name': poolname + } + rc, errordesc, data = self._exec_eternus_cli( + 'show_pool_provision', **param_dict) + + if rc != 0: + msg = (_('_find_pools, show_pool_provision, ' + 'pool name: %(pool_name)s, ' + 'Return code: %(rc)lu, ' + 'Error: %(errordesc)s, ' + 'Message: %(job)s.') + % {'pool_name': poolname, + 'rc': rc, + 'errordesc': errordesc, + 'job': data}) + LOG.error(msg) + raise exception.VolumeBackendAPIException(data=msg) + + pool.provisioned_capacity_gb = data + poolinfo = self.create_pool_info(pool, volume_count, ptype) target_poolname.remove(poolname) @@ -2296,3 +2323,180 @@ class FJDXCommon(object): 'target_pool: %(target_pool)s.', {'poolname': poolname, 'target_pool': target_pool}) return poolname, target_pool + + def _check_user(self): + """Check whether user's role is accessible to ETERNUS and Software.""" + ret = True + rc, errordesc, job = self._exec_eternus_cli('check_user_role') + if rc != 0: + msg = (_('_check_user, ' + 'Return code: %(rc)lu, ' + 'Error: %(errordesc)s, ' + 'Message: %(job)s.') + % {'rc': rc, + 'errordesc': errordesc, + 'job': job}) + LOG.error(msg) + raise exception.VolumeBackendAPIException(data=msg) + + if job != 'Software': + msg = (_('_check_user, ' + 'Specified user(%(user)s) does not have ' + 'Software role: %(role)s.') + % {'user': self._get_drvcfg('EternusUser'), + 'role': job}) + LOG.error(msg) + raise exception.VolumeBackendAPIException(data=msg) + + return ret + + def _exec_eternus_cli(self, command, retry=CONSTANTS.TIMES_MIN, + retry_interval=CONSTANTS.RETRY_INTERVAL, + retry_code=[32787], filename=None, timeout=None, + **param_dict): + """Execute ETERNUS CLI.""" + LOG.debug('_exec_eternus_cli, ' + 'command: %(a)s, ' + 'filename: %(f)s, ' + 'timeout: %(t)s, ' + 'parameters: %(b)s.', + {'a': command, + 'f': filename, + 't': timeout, + 'b': param_dict}) + + result = None + rc = None + retdata = None + errordesc = None + filename = self.configuration.cinder_eternus_config_file + storage_ip = self._get_drvcfg('EternusIP') + if not self.fjdxcli.get(filename): + user = self._get_drvcfg('EternusUser') + password = self._get_drvcfg('EternusPassword') + self.fjdxcli[filename] = ( + eternus_dx_cli.FJDXCLI(user, storage_ip, + password=password)) + + for retry_num in range(retry): + # Execute ETERNUS CLI and get return value. + try: + out_dict = self.fjdxcli[filename].done(command, **param_dict) + result = out_dict.get('result') + rc_str = out_dict.get('rc') + retdata = out_dict.get('message') + except Exception as ex: + msg = (_('_exec_eternus_cli, ' + 'unexpected error: %(ex)s.') + % {'ex': ex}) + LOG.error(msg) + raise exception.VolumeBackendAPIException(data=msg) + + # Check ssh result. + if result == 255: + LOG.info('_exec_eternus_cli, retry, ' + 'command: %(command)s, ' + 'option: %(option)s, ' + 'ip: %(ip)s, ' + 'SSH Result: %(result)s, ' + 'retdata: %(retdata)s, ' + 'TryNum: %(rn)s.', + {'command': command, + 'option': param_dict, + 'ip': storage_ip, + 'result': result, + 'retdata': retdata, + 'rn': (retry_num + 1)}) + time.sleep(retry_interval) + continue + elif result != 0: + msg = (_('_exec_eternus_cli, ' + 'unexpected error, ' + 'command: %(command)s, ' + 'option: %(option)s, ' + 'ip: %(ip)s, ' + 'resuslt: %(result)s, ' + 'retdata: %(retdata)s.') + % {'command': command, + 'option': param_dict, + 'ip': storage_ip, + 'result': result, + 'retdata': retdata}) + LOG.error(msg) + raise exception.VolumeBackendAPIException(data=msg) + + # Check CLI return code. + if rc_str.isdigit(): + # SMI-S style return code. + rc = int(rc_str) + + try: + errordesc = CONSTANTS.RETCODE_dic[str(rc)] + except Exception: + errordesc = 'Undefined Error!!' + + if rc in retry_code: + LOG.info('_exec_eternus_cli, retry, ' + 'ip: %(ip)s, ' + 'RetryCode: %(rc)s, ' + 'TryNum: %(rn)s.', + {'ip': storage_ip, + 'rc': rc, + 'rn': (retry_num + 1)}) + time.sleep(retry_interval) + continue + if rc == 4: + if ('Authentication failed' in retdata and + retry_num + 1 < retry): + LOG.warning('_exec_eternus_cli, retry, ip: %(ip)s, ' + 'Message: %(message)s, ' + 'TryNum: %(rn)s.', + {'ip': storage_ip, + 'message': retdata, + 'rn': (retry_num + 1)}) + time.sleep(1) + continue + + break + else: + # CLI style return code. + LOG.warning('_exec_eternus_cli, ' + 'WARNING!! ' + 'ip: %(ip)s, ' + 'ReturnCode: %(rc_str)s, ' + 'ReturnData: %(retdata)s.', + {'ip': storage_ip, + 'rc_str': rc_str, + 'retdata': retdata}) + + errordesc = rc_str + rc = 4 # Failed. + break + else: + if 0 < result: + msg = (_('_exec_eternus_cli, ' + 'cannot connect to ETERNUS. ' + 'SSH Result: %(result)s, ' + 'retdata: %(retdata)s.') + % {'result': result, + 'retdata': retdata}) + + LOG.error(msg) + raise exception.VolumeBackendAPIException(data=msg) + else: + LOG.warning('_exec_eternus_cli, Retry was exceeded.') + + ret = (rc, errordesc, retdata) + + LOG.debug('_exec_eternus_cli, ' + 'command: %(a)s, ' + 'parameters: %(b)s, ' + 'ip: %(ip)s, ' + 'Return code: %(rc)s, ' + 'Error: %(errordesc)s.', + {'a': command, + 'b': param_dict, + 'ip': storage_ip, + 'rc': rc, + 'errordesc': errordesc}) + return ret diff --git a/doc/source/reference/support-matrix.ini b/doc/source/reference/support-matrix.ini index 37db869c356..210291e12f2 100644 --- a/doc/source/reference/support-matrix.ini +++ b/doc/source/reference/support-matrix.ini @@ -558,7 +558,7 @@ driver.dell_emc_vmax_3=complete driver.dell_emc_vnx=complete driver.dell_emc_vxflexos=complete driver.dell_emc_xtremio=complete -driver.fujitsu_eternus=missing +driver.fujitsu_eternus=complete driver.hpe_3par=complete driver.hpe_lefthand=complete driver.hpe_msa=missing