PY3: Fix file open to read/write the data as bytes
Because swiftclient.put_object assumes it as bytes object. Only ZipFile is not suppoting rb mode so leave it as it is. It should be okay because Storlets use it just to pick up the filnames that should be string instead of bytes. Change-Id: I834b29886a18269de2745ffe584291491e80f9eb
This commit is contained in:
parent
77573c5906
commit
84421c93d0
@ -138,8 +138,8 @@ the code below shows the invocation. Some notes:
|
||||
querystring = 'execute=true'
|
||||
else:
|
||||
querystring = None
|
||||
|
||||
fileobj = open(file_name_to_upload,'r')
|
||||
|
||||
fileobj = open(file_name_to_upload,'rb')
|
||||
random_md = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(32))
|
||||
headers = {'X-Run-Storlet': 'identitystorlet-1.0.jar', 'X-Object-Meta-Testkey' : random_md}
|
||||
c.put_object(url,
|
||||
|
@ -232,7 +232,7 @@ in the deployment using Swift client section above.
|
||||
'X-Object-Meta-Storlet-Dependency': dependencies,
|
||||
'X-Object-Meta-Storlet-Object-Metadata':'no',
|
||||
'X-Object-Meta-Storlet-Main': main_class_name}
|
||||
f = open('%s/%s' % (local_path_to_storlet, storlet_name), 'r')
|
||||
f = open('%s/%s' % (local_path_to_storlet, storlet_name), 'rb')
|
||||
content_length = None
|
||||
response = dict()
|
||||
client.put_object(url, token, 'storlet', storlet_name, f,
|
||||
@ -248,7 +248,7 @@ in the deployment using Swift client section above.
|
||||
metadata = {'X-Object-Meta-Storlet-Dependency-Version': '1'}
|
||||
# for an executable dependency
|
||||
# metadata['X-Object-Meta-Storlet-Dependency-Permissions'] = '0755'
|
||||
f = open('%s/%s'% (local_path_to_dependency, dependency_name), 'r')
|
||||
f = open('%s/%s'% (local_path_to_dependency, dependency_name), 'rb')
|
||||
content_length = None
|
||||
response = dict()
|
||||
client.put_object(url, token, 'dependency', dependency_name, f,
|
||||
|
@ -121,7 +121,7 @@ class SBusServer(object):
|
||||
|
||||
try:
|
||||
outfd = dtg.service_out_fd
|
||||
with os.fdopen(outfd, 'w') as outfile:
|
||||
with os.fdopen(outfd, 'wb') as outfile:
|
||||
self._respond(outfile, resp)
|
||||
except AttributeError:
|
||||
# TODO(takashi): Currently we return response via service out fd
|
||||
|
@ -81,12 +81,12 @@ class StorletFile(object):
|
||||
|
||||
|
||||
class StorletOutputFile(StorletFile):
|
||||
mode = 'w'
|
||||
mode = 'wb'
|
||||
|
||||
def __init__(self, md_fd, obj_fd):
|
||||
super(StorletOutputFile, self).__init__(obj_fd)
|
||||
self._metadata = None
|
||||
self.md_file = os.fdopen(md_fd, 'w')
|
||||
self.md_file = os.fdopen(md_fd, 'wb')
|
||||
|
||||
def get_metadata(self):
|
||||
if self._metadata is None:
|
||||
@ -96,7 +96,7 @@ class StorletOutputFile(StorletFile):
|
||||
def set_metadata(self, md):
|
||||
if self.md_file.closed:
|
||||
raise IOError('Sending metadata twice is not allowed')
|
||||
self.md_file.write(json.dumps(md))
|
||||
self.md_file.write(json.dumps(md).encode('utf-8'))
|
||||
self._metadata = md
|
||||
self.md_file.close()
|
||||
|
||||
@ -201,7 +201,7 @@ class StorletLogger(object):
|
||||
def __init__(self, storlet_name, fd):
|
||||
self.fd = fd
|
||||
self.storlet_name = storlet_name
|
||||
self.log_file = os.fdopen(self.fd, 'w')
|
||||
self.log_file = os.fdopen(self.fd, 'wb')
|
||||
|
||||
@property
|
||||
def closed(self):
|
||||
|
@ -157,8 +157,8 @@ class StorletDaemon(SBusServer):
|
||||
self._wait_child_process()
|
||||
|
||||
self.logger.debug('Returning task_id: %s ' % task_id)
|
||||
with os.fdopen(task_id_out_fd, 'w') as outfile:
|
||||
outfile.write(task_id)
|
||||
with os.fdopen(task_id_out_fd, 'wb') as outfile:
|
||||
outfile.write(task_id.encode("utf-8"))
|
||||
|
||||
storlet_md = dtg.object_in_storlet_metadata
|
||||
params = dtg.params
|
||||
|
@ -138,7 +138,7 @@ class StorletDaemonFactory(SBusServer):
|
||||
# TODO(takashi): We had better use contextmanager
|
||||
# TODO(takashi): Where is this closed?
|
||||
try:
|
||||
dn = open(os.devnull, 'w')
|
||||
dn = open(os.devnull, 'wb')
|
||||
daemon_p = subprocess.Popen(
|
||||
pargs, stdout=dn, stderr=subprocess.PIPE,
|
||||
close_fds=True, shell=False, env=env)
|
||||
|
@ -249,7 +249,7 @@ class StorletGatewayDocker(StorletGatewayBase):
|
||||
:params sreq: DockerStorletRequest instance
|
||||
"""
|
||||
if sreq.generate_log:
|
||||
with open(slog_path, 'r') as logfile:
|
||||
with open(slog_path, 'rb') as logfile:
|
||||
storlet_name = sreq.storlet_id.split('-')[0]
|
||||
log_obj_name = '%s.log' % storlet_name
|
||||
sreq.file_manager.put_log(log_obj_name, logfile)
|
||||
@ -315,7 +315,7 @@ class StorletGatewayDocker(StorletGatewayBase):
|
||||
data_iter, perm = get_func(obj_name)
|
||||
|
||||
# TODO(takashi): Do not directly write to target path
|
||||
with open(cache_target_path, 'w') as fn:
|
||||
with open(cache_target_path, 'wb') as fn:
|
||||
for data in data_iter:
|
||||
fn.write(data)
|
||||
|
||||
|
@ -774,7 +774,7 @@ class StorletInvocationProtocol(object):
|
||||
|
||||
@contextmanager
|
||||
def _open_writer(self, fd):
|
||||
with os.fdopen(fd, 'w') as writer:
|
||||
with os.fdopen(fd, 'wb') as writer:
|
||||
yield writer
|
||||
|
||||
def _write_input_data(self, fd, data_iter):
|
||||
|
@ -432,7 +432,7 @@ class StorletMagics(Magics):
|
||||
# invoke storlet app on copy
|
||||
conn = get_swift_connection()
|
||||
response_dict = dict()
|
||||
with open(args.input, 'r') as content:
|
||||
with open(args.input, 'rb') as content:
|
||||
conn.put_object(
|
||||
dst_container, dst_obj,
|
||||
content,
|
||||
|
@ -29,7 +29,7 @@ def put_local_file(url, token, container, local_dir, local_file, headers=None):
|
||||
:param headers: headers parameters to be included in request headers
|
||||
"""
|
||||
resp = dict()
|
||||
with open(os.path.join(local_dir, local_file), 'r') as f:
|
||||
with open(os.path.join(local_dir, local_file), 'rb') as f:
|
||||
client.put_object(url, token, container, local_file, f,
|
||||
headers=headers,
|
||||
content_type="application/octet-stream",
|
||||
|
@ -60,6 +60,10 @@ class DeployTestMixin(object):
|
||||
except pexpect.EOF as err:
|
||||
self.fail(
|
||||
'Expected message "%s" not found: %s' % (message, err))
|
||||
except AssertionError:
|
||||
# drain all error message lines for debug
|
||||
print(child.read())
|
||||
raise
|
||||
|
||||
# Make sure the existence in the swift inside
|
||||
# for storlet app
|
||||
|
@ -27,9 +27,11 @@ import unittest
|
||||
def create_local_chunks():
|
||||
for i in range(1, 10):
|
||||
oname = '/tmp/slo_chunk_%d' % i
|
||||
f = open(oname, 'w')
|
||||
f.write(''.join(random.choice(string.ascii_uppercase + string.digits)
|
||||
for _ in range(1048576)))
|
||||
f = open(oname, 'wb')
|
||||
f.write(b''.join(
|
||||
random.choice(
|
||||
string.ascii_uppercase + string.digits).encode("utf-8")
|
||||
for _ in range(1048576)))
|
||||
f.close()
|
||||
|
||||
|
||||
@ -70,7 +72,7 @@ class TestSLO(StorletJavaFunctionalTest):
|
||||
i = 1
|
||||
for chunk in body:
|
||||
oname = '/tmp/slo_chunk_%d' % i
|
||||
f = open(oname, 'r')
|
||||
f = open(oname, 'rb')
|
||||
file_content = f.read()
|
||||
# print '%s %s' % (chunk[:10], file_content[:10])
|
||||
# print '%d %d' % (len(chunk), len(file_content))
|
||||
@ -83,7 +85,7 @@ class TestSLO(StorletJavaFunctionalTest):
|
||||
assembly = []
|
||||
for i in range(1, 10):
|
||||
oname = '/tmp/slo_chunk_%d' % i
|
||||
f = open(oname, 'r')
|
||||
f = open(oname, 'rb')
|
||||
content_length = None
|
||||
response = dict()
|
||||
c.put_object(self.url, self.token,
|
||||
@ -121,7 +123,7 @@ class TestSLO(StorletJavaFunctionalTest):
|
||||
if chunk:
|
||||
if i < 10:
|
||||
oname = '/tmp/slo_chunk_%d' % i
|
||||
f = open(oname, 'r')
|
||||
f = open(oname, 'rb')
|
||||
file_content = f.read()
|
||||
# print '%s %s' % (chunk[:10], file_content[:10])
|
||||
# print '%d %d' % (len(chunk), len(file_content))
|
||||
@ -132,7 +134,7 @@ class TestSLO(StorletJavaFunctionalTest):
|
||||
aux_content = ''
|
||||
for j in range(1, 4):
|
||||
oname = '/tmp/aux_file%d' % j
|
||||
f = open(oname, 'r')
|
||||
f = open(oname, 'rb')
|
||||
aux_content += f.read()
|
||||
f.close()
|
||||
self.ssertEqual(chunk, aux_content)
|
||||
|
@ -128,15 +128,15 @@ class TestIdentityStorlet(StorletJavaFunctionalTest):
|
||||
def test_put_1GB_file(self):
|
||||
gf_file_path = '/tmp/1GB_file'
|
||||
with self._filecontext(gf_file_path):
|
||||
GBFile = open('/tmp/1GB_file', 'w')
|
||||
GBFile = open('/tmp/1GB_file', 'wb')
|
||||
for _ in range(128):
|
||||
uploaded_content = ''.join('1' for _ in range(8 * 1024 * 1024))
|
||||
GBFile.write(uploaded_content)
|
||||
GBFile.write(uploaded_content.encode("utf-8"))
|
||||
GBFile.close()
|
||||
|
||||
headers = {'X-Run-Storlet': self.storlet_name}
|
||||
headers.update(self.additional_headers)
|
||||
GBFile = open('/tmp/1GB_file', 'r')
|
||||
GBFile = open('/tmp/1GB_file', 'rb')
|
||||
response = dict()
|
||||
c.put_object(self.url, self.token,
|
||||
self.container, '1GBFile', GBFile,
|
||||
|
@ -39,7 +39,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
|
||||
self.storlet_file,
|
||||
response_dict=resp,
|
||||
headers=headers)
|
||||
with open('/tmp/sample.jpg', 'w') as f:
|
||||
with open('/tmp/sample.jpg', 'wb') as f:
|
||||
f.write(gf)
|
||||
|
||||
self.assertIn(resp['status'], [200, 202])
|
||||
@ -50,7 +50,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
|
||||
headers.update(self.additional_headers)
|
||||
resp = dict()
|
||||
source_file = '%s/%s' % (self.path_to_bundle, self.storlet_file)
|
||||
with open(source_file, 'r') as f:
|
||||
with open(source_file, 'rb') as f:
|
||||
c.put_object(self.url, self.token,
|
||||
self.container, 'gen_thumb_on_put.jpg', f,
|
||||
headers=headers,
|
||||
|
@ -97,10 +97,10 @@ class TestSLO(StorletPythonFunctionalTest):
|
||||
if i in range(9):
|
||||
self.assertEqual(chunk, self.chunks[i])
|
||||
else:
|
||||
aux_content = ''
|
||||
aux_content = b''
|
||||
for j in range(1, 4):
|
||||
oname = 'aux_file%d' % j
|
||||
with open(oname, 'r') as f:
|
||||
with open(oname, 'rb') as f:
|
||||
aux_content += f.read()
|
||||
self.asertEqual(chunk, aux_content)
|
||||
|
||||
|
@ -83,13 +83,13 @@ class TestSimpleStorlet(StorletPythonFunctionalTest):
|
||||
@attr('slow')
|
||||
def test_put_512MB_file(self):
|
||||
with tempfile.NamedTemporaryFile() as f:
|
||||
with open(f.name, 'w') as wf:
|
||||
with open(f.name, 'wb') as wf:
|
||||
for _ in range(128):
|
||||
wf.write('1' * (4 * 1024 * 1024))
|
||||
wf.write(b'1' * (4 * 1024 * 1024))
|
||||
|
||||
headers = {'X-Run-Storlet': self.storlet_name}
|
||||
headers.update(self.additional_headers)
|
||||
with open(f.name, 'r') as rf:
|
||||
with open(f.name, 'rb') as rf:
|
||||
response = dict()
|
||||
client.put_object(self.url, self.token,
|
||||
self.container, f.name, rf,
|
||||
|
@ -98,20 +98,20 @@ class TestStorletOutputFile(TestStorletFile):
|
||||
def test_write(self):
|
||||
with self.sfile as sfile:
|
||||
sfile.set_metadata({})
|
||||
sfile.write('testing')
|
||||
sfile.write(b'testing')
|
||||
sfile.flush()
|
||||
|
||||
with open(self.fname, 'r') as f:
|
||||
self.assertEqual('testing', f.read())
|
||||
with open(self.fname, 'rb') as f:
|
||||
self.assertEqual(b'testing', f.read())
|
||||
|
||||
def test_writelines(self):
|
||||
with self.sfile as sfile:
|
||||
sfile.set_metadata({})
|
||||
sfile.writelines(['tes', 'ti', 'ng'])
|
||||
sfile.writelines([b'tes', b'ti', b'ng'])
|
||||
sfile.flush()
|
||||
|
||||
with open(self.fname, 'r') as f:
|
||||
self.assertEqual('testing', f.read())
|
||||
with open(self.fname, 'rb') as f:
|
||||
self.assertEqual(b'testing', f.read())
|
||||
|
||||
|
||||
class TestStorletInputFile(TestStorletFile):
|
||||
|
@ -19,7 +19,7 @@ from shutil import rmtree
|
||||
from tempfile import mkdtemp
|
||||
import eventlet
|
||||
import json
|
||||
from six import StringIO
|
||||
from six import BytesIO, StringIO
|
||||
|
||||
import mock
|
||||
import unittest
|
||||
@ -231,8 +231,8 @@ class TestStorletDockerGateway(unittest.TestCase):
|
||||
# TODO(kota_): should be 'storlet-internal-client.conf' actually
|
||||
ic_conf_path = os.path.join(self.tempdir,
|
||||
'storlet-proxy-server.conf')
|
||||
with open(ic_conf_path, 'w') as f:
|
||||
f.write("""
|
||||
with open(ic_conf_path, 'wb') as f:
|
||||
f.write(b"""
|
||||
[DEFAULT]
|
||||
[pipeline:main]
|
||||
pipeline = catch_errors proxy-logging cache proxy-server
|
||||
@ -519,10 +519,10 @@ use = egg:swift#catch_errors
|
||||
|
||||
class MockFileManager(object):
|
||||
def get_storlet(self, req):
|
||||
return StringIO('mock'), None
|
||||
return BytesIO(b'mock'), None
|
||||
|
||||
def get_dependency(self, req):
|
||||
return StringIO('mock'), None
|
||||
return BytesIO(b'mock'), None
|
||||
|
||||
st_req.file_manager = MockFileManager()
|
||||
|
||||
|
Loading…
x
Reference in New Issue
Block a user