PY3: Fix file open to read/write the data as bytes

Because swiftclient.put_object assumes it as bytes object.
Only ZipFile is not suppoting rb mode so leave it as it is.
It should be okay because Storlets use it just to pick up
the filnames that should be string instead of bytes.

Change-Id: I834b29886a18269de2745ffe584291491e80f9eb
This commit is contained in:
Kota Tsuyuzaki 2019-11-01 18:44:29 +09:00 committed by Tim Burke
parent 77573c5906
commit 84421c93d0
18 changed files with 51 additions and 45 deletions

View File

@ -138,8 +138,8 @@ the code below shows the invocation. Some notes:
querystring = 'execute=true'
else:
querystring = None
fileobj = open(file_name_to_upload,'r')
fileobj = open(file_name_to_upload,'rb')
random_md = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(32))
headers = {'X-Run-Storlet': 'identitystorlet-1.0.jar', 'X-Object-Meta-Testkey' : random_md}
c.put_object(url,

View File

@ -232,7 +232,7 @@ in the deployment using Swift client section above.
'X-Object-Meta-Storlet-Dependency': dependencies,
'X-Object-Meta-Storlet-Object-Metadata':'no',
'X-Object-Meta-Storlet-Main': main_class_name}
f = open('%s/%s' % (local_path_to_storlet, storlet_name), 'r')
f = open('%s/%s' % (local_path_to_storlet, storlet_name), 'rb')
content_length = None
response = dict()
client.put_object(url, token, 'storlet', storlet_name, f,
@ -248,7 +248,7 @@ in the deployment using Swift client section above.
metadata = {'X-Object-Meta-Storlet-Dependency-Version': '1'}
# for an executable dependency
# metadata['X-Object-Meta-Storlet-Dependency-Permissions'] = '0755'
f = open('%s/%s'% (local_path_to_dependency, dependency_name), 'r')
f = open('%s/%s'% (local_path_to_dependency, dependency_name), 'rb')
content_length = None
response = dict()
client.put_object(url, token, 'dependency', dependency_name, f,

View File

@ -121,7 +121,7 @@ class SBusServer(object):
try:
outfd = dtg.service_out_fd
with os.fdopen(outfd, 'w') as outfile:
with os.fdopen(outfd, 'wb') as outfile:
self._respond(outfile, resp)
except AttributeError:
# TODO(takashi): Currently we return response via service out fd

View File

@ -81,12 +81,12 @@ class StorletFile(object):
class StorletOutputFile(StorletFile):
mode = 'w'
mode = 'wb'
def __init__(self, md_fd, obj_fd):
super(StorletOutputFile, self).__init__(obj_fd)
self._metadata = None
self.md_file = os.fdopen(md_fd, 'w')
self.md_file = os.fdopen(md_fd, 'wb')
def get_metadata(self):
if self._metadata is None:
@ -96,7 +96,7 @@ class StorletOutputFile(StorletFile):
def set_metadata(self, md):
if self.md_file.closed:
raise IOError('Sending metadata twice is not allowed')
self.md_file.write(json.dumps(md))
self.md_file.write(json.dumps(md).encode('utf-8'))
self._metadata = md
self.md_file.close()
@ -201,7 +201,7 @@ class StorletLogger(object):
def __init__(self, storlet_name, fd):
self.fd = fd
self.storlet_name = storlet_name
self.log_file = os.fdopen(self.fd, 'w')
self.log_file = os.fdopen(self.fd, 'wb')
@property
def closed(self):

View File

@ -157,8 +157,8 @@ class StorletDaemon(SBusServer):
self._wait_child_process()
self.logger.debug('Returning task_id: %s ' % task_id)
with os.fdopen(task_id_out_fd, 'w') as outfile:
outfile.write(task_id)
with os.fdopen(task_id_out_fd, 'wb') as outfile:
outfile.write(task_id.encode("utf-8"))
storlet_md = dtg.object_in_storlet_metadata
params = dtg.params

View File

@ -138,7 +138,7 @@ class StorletDaemonFactory(SBusServer):
# TODO(takashi): We had better use contextmanager
# TODO(takashi): Where is this closed?
try:
dn = open(os.devnull, 'w')
dn = open(os.devnull, 'wb')
daemon_p = subprocess.Popen(
pargs, stdout=dn, stderr=subprocess.PIPE,
close_fds=True, shell=False, env=env)

View File

@ -249,7 +249,7 @@ class StorletGatewayDocker(StorletGatewayBase):
:params sreq: DockerStorletRequest instance
"""
if sreq.generate_log:
with open(slog_path, 'r') as logfile:
with open(slog_path, 'rb') as logfile:
storlet_name = sreq.storlet_id.split('-')[0]
log_obj_name = '%s.log' % storlet_name
sreq.file_manager.put_log(log_obj_name, logfile)
@ -315,7 +315,7 @@ class StorletGatewayDocker(StorletGatewayBase):
data_iter, perm = get_func(obj_name)
# TODO(takashi): Do not directly write to target path
with open(cache_target_path, 'w') as fn:
with open(cache_target_path, 'wb') as fn:
for data in data_iter:
fn.write(data)

View File

@ -774,7 +774,7 @@ class StorletInvocationProtocol(object):
@contextmanager
def _open_writer(self, fd):
with os.fdopen(fd, 'w') as writer:
with os.fdopen(fd, 'wb') as writer:
yield writer
def _write_input_data(self, fd, data_iter):

View File

@ -432,7 +432,7 @@ class StorletMagics(Magics):
# invoke storlet app on copy
conn = get_swift_connection()
response_dict = dict()
with open(args.input, 'r') as content:
with open(args.input, 'rb') as content:
conn.put_object(
dst_container, dst_obj,
content,

View File

@ -29,7 +29,7 @@ def put_local_file(url, token, container, local_dir, local_file, headers=None):
:param headers: headers parameters to be included in request headers
"""
resp = dict()
with open(os.path.join(local_dir, local_file), 'r') as f:
with open(os.path.join(local_dir, local_file), 'rb') as f:
client.put_object(url, token, container, local_file, f,
headers=headers,
content_type="application/octet-stream",

View File

@ -60,6 +60,10 @@ class DeployTestMixin(object):
except pexpect.EOF as err:
self.fail(
'Expected message "%s" not found: %s' % (message, err))
except AssertionError:
# drain all error message lines for debug
print(child.read())
raise
# Make sure the existence in the swift inside
# for storlet app

View File

@ -27,9 +27,11 @@ import unittest
def create_local_chunks():
for i in range(1, 10):
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'w')
f.write(''.join(random.choice(string.ascii_uppercase + string.digits)
for _ in range(1048576)))
f = open(oname, 'wb')
f.write(b''.join(
random.choice(
string.ascii_uppercase + string.digits).encode("utf-8")
for _ in range(1048576)))
f.close()
@ -70,7 +72,7 @@ class TestSLO(StorletJavaFunctionalTest):
i = 1
for chunk in body:
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'r')
f = open(oname, 'rb')
file_content = f.read()
# print '%s %s' % (chunk[:10], file_content[:10])
# print '%d %d' % (len(chunk), len(file_content))
@ -83,7 +85,7 @@ class TestSLO(StorletJavaFunctionalTest):
assembly = []
for i in range(1, 10):
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'r')
f = open(oname, 'rb')
content_length = None
response = dict()
c.put_object(self.url, self.token,
@ -121,7 +123,7 @@ class TestSLO(StorletJavaFunctionalTest):
if chunk:
if i < 10:
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'r')
f = open(oname, 'rb')
file_content = f.read()
# print '%s %s' % (chunk[:10], file_content[:10])
# print '%d %d' % (len(chunk), len(file_content))
@ -132,7 +134,7 @@ class TestSLO(StorletJavaFunctionalTest):
aux_content = ''
for j in range(1, 4):
oname = '/tmp/aux_file%d' % j
f = open(oname, 'r')
f = open(oname, 'rb')
aux_content += f.read()
f.close()
self.ssertEqual(chunk, aux_content)

View File

@ -128,15 +128,15 @@ class TestIdentityStorlet(StorletJavaFunctionalTest):
def test_put_1GB_file(self):
gf_file_path = '/tmp/1GB_file'
with self._filecontext(gf_file_path):
GBFile = open('/tmp/1GB_file', 'w')
GBFile = open('/tmp/1GB_file', 'wb')
for _ in range(128):
uploaded_content = ''.join('1' for _ in range(8 * 1024 * 1024))
GBFile.write(uploaded_content)
GBFile.write(uploaded_content.encode("utf-8"))
GBFile.close()
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
GBFile = open('/tmp/1GB_file', 'r')
GBFile = open('/tmp/1GB_file', 'rb')
response = dict()
c.put_object(self.url, self.token,
self.container, '1GBFile', GBFile,

View File

@ -39,7 +39,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
self.storlet_file,
response_dict=resp,
headers=headers)
with open('/tmp/sample.jpg', 'w') as f:
with open('/tmp/sample.jpg', 'wb') as f:
f.write(gf)
self.assertIn(resp['status'], [200, 202])
@ -50,7 +50,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
headers.update(self.additional_headers)
resp = dict()
source_file = '%s/%s' % (self.path_to_bundle, self.storlet_file)
with open(source_file, 'r') as f:
with open(source_file, 'rb') as f:
c.put_object(self.url, self.token,
self.container, 'gen_thumb_on_put.jpg', f,
headers=headers,

View File

@ -97,10 +97,10 @@ class TestSLO(StorletPythonFunctionalTest):
if i in range(9):
self.assertEqual(chunk, self.chunks[i])
else:
aux_content = ''
aux_content = b''
for j in range(1, 4):
oname = 'aux_file%d' % j
with open(oname, 'r') as f:
with open(oname, 'rb') as f:
aux_content += f.read()
self.asertEqual(chunk, aux_content)

View File

@ -83,13 +83,13 @@ class TestSimpleStorlet(StorletPythonFunctionalTest):
@attr('slow')
def test_put_512MB_file(self):
with tempfile.NamedTemporaryFile() as f:
with open(f.name, 'w') as wf:
with open(f.name, 'wb') as wf:
for _ in range(128):
wf.write('1' * (4 * 1024 * 1024))
wf.write(b'1' * (4 * 1024 * 1024))
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
with open(f.name, 'r') as rf:
with open(f.name, 'rb') as rf:
response = dict()
client.put_object(self.url, self.token,
self.container, f.name, rf,

View File

@ -98,20 +98,20 @@ class TestStorletOutputFile(TestStorletFile):
def test_write(self):
with self.sfile as sfile:
sfile.set_metadata({})
sfile.write('testing')
sfile.write(b'testing')
sfile.flush()
with open(self.fname, 'r') as f:
self.assertEqual('testing', f.read())
with open(self.fname, 'rb') as f:
self.assertEqual(b'testing', f.read())
def test_writelines(self):
with self.sfile as sfile:
sfile.set_metadata({})
sfile.writelines(['tes', 'ti', 'ng'])
sfile.writelines([b'tes', b'ti', b'ng'])
sfile.flush()
with open(self.fname, 'r') as f:
self.assertEqual('testing', f.read())
with open(self.fname, 'rb') as f:
self.assertEqual(b'testing', f.read())
class TestStorletInputFile(TestStorletFile):

View File

@ -19,7 +19,7 @@ from shutil import rmtree
from tempfile import mkdtemp
import eventlet
import json
from six import StringIO
from six import BytesIO, StringIO
import mock
import unittest
@ -231,8 +231,8 @@ class TestStorletDockerGateway(unittest.TestCase):
# TODO(kota_): should be 'storlet-internal-client.conf' actually
ic_conf_path = os.path.join(self.tempdir,
'storlet-proxy-server.conf')
with open(ic_conf_path, 'w') as f:
f.write("""
with open(ic_conf_path, 'wb') as f:
f.write(b"""
[DEFAULT]
[pipeline:main]
pipeline = catch_errors proxy-logging cache proxy-server
@ -519,10 +519,10 @@ use = egg:swift#catch_errors
class MockFileManager(object):
def get_storlet(self, req):
return StringIO('mock'), None
return BytesIO(b'mock'), None
def get_dependency(self, req):
return StringIO('mock'), None
return BytesIO(b'mock'), None
st_req.file_manager = MockFileManager()