openstack/swift
Code Issues Proposed changes

py3: port bufferedhttp (hopefully)

Browse Source 
I've at least tried it out with a py3 proxy, and it seems to work out
OK. I haven't tried killing the socket and verifying that it's actualy
dead, but getting a hold of _real_close *seems like* what we want?

At least the three (!!) tests pass.

Change-Id: Ic08c26185d63a36a5422793d81f621e0698fa572
This commit is contained in:
Tim Burke 2018-03-01 00:29:24 +00:00 committed by Christian Schwede
parent e547448e96
commit df954e2709
3 changed files with 59 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
swift/common/bufferedhttp.py
View File

@ -51,15 +51,19 @@ class BufferedHTTPResponse(HTTPResponse):
method=None): # pragma: no cover method=None): # pragma: no cover
self.sock = sock self.sock = sock
# sock is an eventlet.greenio.GreenSocket # sock is an eventlet.greenio.GreenSocket
if six.PY2:
# sock.fd is a socket._socketobject # sock.fd is a socket._socketobject
# sock.fd._sock is a socket._socket object, which is what we want. # sock.fd._sock is a _socket.socket object, which is what we want.
self._real_socket = sock.fd._sock self._real_socket = sock.fd._sock
else:
# sock.fd is a socket.socket, which should have a _real_close
self._real_socket = sock.fd
self.fp = sock.makefile('rb') self.fp = sock.makefile('rb')
self.debuglevel = debuglevel self.debuglevel = debuglevel
self.strict = strict self.strict = strict
self._method = method self._method = method
self.msg = None self.headers = self.msg = None
# from the Status-Line of the response # from the Status-Line of the response
self.version = _UNKNOWN # HTTP-Version self.version = _UNKNOWN # HTTP-Version
@ -70,7 +74,7 @@ class BufferedHTTPResponse(HTTPResponse):
self.chunk_left = _UNKNOWN # bytes left to read in current chunk self.chunk_left = _UNKNOWN # bytes left to read in current chunk
self.length = _UNKNOWN # number of bytes left in response self.length = _UNKNOWN # number of bytes left in response
self.will_close = _UNKNOWN # conn will close at end of response self.will_close = _UNKNOWN # conn will close at end of response
self._readline_buffer = '' self._readline_buffer = b''
def expect_response(self): def expect_response(self):
if self.fp: if self.fp:
@ -85,8 +89,15 @@ class BufferedHTTPResponse(HTTPResponse):
self.status = status self.status = status
self.reason = reason.strip() self.reason = reason.strip()
self.version = 11 self.version = 11
if six.PY2:
# Under py2, HTTPMessage.__init__ reads the headers
# which advances fp
self.msg = HTTPMessage(self.fp, 0) self.msg = HTTPMessage(self.fp, 0)
# immediately kill msg.fp to make sure it isn't read again
self.msg.fp = None self.msg.fp = None
else:
# py3 has a separate helper for it
self.headers = self.msg = httplib.parse_headers(self.fp)
def read(self, amt=None): def read(self, amt=None):
if not self._readline_buffer: if not self._readline_buffer:
@ -96,7 +107,7 @@ class BufferedHTTPResponse(HTTPResponse):
# Unbounded read: send anything we have buffered plus whatever # Unbounded read: send anything we have buffered plus whatever
# is left. # is left.
buffered = self._readline_buffer buffered = self._readline_buffer
self._readline_buffer = '' self._readline_buffer = b''
return buffered + HTTPResponse.read(self, amt) return buffered + HTTPResponse.read(self, amt)
elif amt <= len(self._readline_buffer): elif amt <= len(self._readline_buffer):
# Bounded read that we can satisfy entirely from our buffer # Bounded read that we can satisfy entirely from our buffer
@ -107,7 +118,7 @@ class BufferedHTTPResponse(HTTPResponse):
# Bounded read that wants more bytes than we have # Bounded read that wants more bytes than we have
smaller_amt = amt - len(self._readline_buffer) smaller_amt = amt - len(self._readline_buffer)
buf = self._readline_buffer buf = self._readline_buffer
self._readline_buffer = '' self._readline_buffer = b''
return buf + HTTPResponse.read(self, smaller_amt) return buf + HTTPResponse.read(self, smaller_amt)
def readline(self, size=1024): def readline(self, size=1024):
@ -118,7 +129,7 @@ class BufferedHTTPResponse(HTTPResponse):
# # too. # # too.
# #
# Yes, it certainly would. # Yes, it certainly would.
while ('\n' not in self._readline_buffer while (b'\n' not in self._readline_buffer
and len(self._readline_buffer) < size): and len(self._readline_buffer) < size):
read_size = size - len(self._readline_buffer) read_size = size - len(self._readline_buffer)
chunk = HTTPResponse.read(self, read_size) chunk = HTTPResponse.read(self, read_size)
@ -126,7 +137,7 @@ class BufferedHTTPResponse(HTTPResponse):
break break
self._readline_buffer += chunk self._readline_buffer += chunk
line, newline, rest = self._readline_buffer.partition('\n') line, newline, rest = self._readline_buffer.partition(b'\n')
self._readline_buffer = rest self._readline_buffer = rest
return line + newline return line + newline
@ -139,9 +150,14 @@ class BufferedHTTPResponse(HTTPResponse):
you care about has a reference to this socket. you care about has a reference to this socket.
""" """
if self._real_socket: if self._real_socket:
# this is idempotent; see sock_close in Modules/socketmodule.c in if six.PY2:
# the Python source for details. # this is idempotent; see sock_close in Modules/socketmodule.c
# in the Python source for details.
self._real_socket.close() self._real_socket.close()
else:
# Hopefully this is equivalent?
# TODO: verify that this does everything ^^^^ does for py2
self._real_socket._real_close()
self._real_socket = None self._real_socket = None
self.close() self.close()
@ -168,8 +184,10 @@ class BufferedHTTPConnection(HTTPConnection):
skip_accept_encoding) skip_accept_encoding)
def getexpect(self): def getexpect(self):
response = BufferedHTTPResponse(self.sock, strict=self.strict, kwargs = {'method': self._method}
method=self._method) if hasattr(self, 'strict'):
kwargs['strict'] = self.strict
response = BufferedHTTPResponse(self.sock, **kwargs)
response.expect_response() response.expect_response()
return response return response
@ -205,7 +223,11 @@ def http_connect(ipaddr, port, device, partition, method, path,
path = path.encode("utf-8") path = path.encode("utf-8")
if isinstance(device, six.text_type): if isinstance(device, six.text_type):
device = device.encode("utf-8") device = device.encode("utf-8")
path = quote('/' + device + '/' + str(partition) + path) if isinstance(partition, six.text_type):
partition = partition.encode('utf-8')
elif isinstance(partition, six.integer_types):
partition = str(partition).encode('ascii')
path = quote(b'/' + device + b'/' + partition + path)
return http_connect_raw( return http_connect_raw(
ipaddr, port, method, path, headers, query_string, ssl) ipaddr, port, method, path, headers, query_string, ssl)

37
test/unit/common/test_bufferedhttp.py
View File

@ -14,9 +14,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import mock import mock
import unittest import unittest
import socket import socket
from eventlet import spawn, Timeout from eventlet import spawn, Timeout
@ -53,28 +51,31 @@ class TestBufferedHTTP(unittest.TestCase):
try: try:
with Timeout(3): with Timeout(3):
sock, addr = bindsock.accept() sock, addr = bindsock.accept()
fp = sock.makefile() fp = sock.makefile('rwb')
fp.write('HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\n' fp.write(b'HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\n'
'RESPONSE') b'RESPONSE')
fp.flush() fp.flush()
line = fp.readline()
path = b'/dev/' + expected_par + b'/path/..%25/?omg&no=%7f'
self.assertEqual( self.assertEqual(
fp.readline(), line,
'PUT /dev/%s/path/..%%25/?omg&no=%%7f HTTP/1.1\r\n' % b'PUT ' + path + b' HTTP/1.1\r\n')
expected_par)
headers = {} headers = {}
line = fp.readline() line = fp.readline()
while line and line != '\r\n': while line and line != b'\r\n':
headers[line.split(':')[0].lower()] = \ headers[line.split(b':')[0].lower()] = \
line.split(':')[1].strip() line.split(b':')[1].strip()
line = fp.readline() line = fp.readline()
self.assertEqual(headers['content-length'], '7') self.assertEqual(headers[b'content-length'], b'7')
self.assertEqual(headers['x-header'], 'value') self.assertEqual(headers[b'x-header'], b'value')
self.assertEqual(fp.readline(), 'REQUEST\r\n') self.assertEqual(fp.readline(), b'REQUEST\r\n')
except BaseException as err: except BaseException as err:
return err return err
return None return None
for par in ('par', 1357): for spawn_par, par in (
event = spawn(accept, par) (b'par', b'par'), (b'up%C3%A5r', u'up\xe5r'),
(b'%C3%BCpar', b'\xc3\xbcpar'), (b'1357', 1357)):
event = spawn(accept, spawn_par)
try: try:
with Timeout(3): with Timeout(3):
conn = bufferedhttp.http_connect( conn = bufferedhttp.http_connect(
@ -83,7 +84,7 @@ class TestBufferedHTTP(unittest.TestCase):
'content-length': 7, 'content-length': 7,
'x-header': 'value'}, 'x-header': 'value'},
query_string='omg&no=%7f') query_string='omg&no=%7f')
conn.send('REQUEST\r\n') conn.send(b'REQUEST\r\n')
self.assertTrue(conn.sock.getsockopt(socket.IPPROTO_TCP, self.assertTrue(conn.sock.getsockopt(socket.IPPROTO_TCP,
socket.TCP_NODELAY)) socket.TCP_NODELAY))
resp = conn.getresponse() resp = conn.getresponse()
@ -91,7 +92,7 @@ class TestBufferedHTTP(unittest.TestCase):
conn.close() conn.close()
self.assertEqual(resp.status, 200) self.assertEqual(resp.status, 200)
self.assertEqual(resp.reason, 'OK') self.assertEqual(resp.reason, 'OK')
self.assertEqual(body, 'RESPONSE') self.assertEqual(body, b'RESPONSE')
finally: finally:
err = event.wait() err = event.wait()
if err: if err:

1
tox.ini
View File

@ -43,6 +43,7 @@ commands =
test/unit/common/middleware/test_healthcheck.py \ test/unit/common/middleware/test_healthcheck.py \
test/unit/common/middleware/test_proxy_logging.py \ test/unit/common/middleware/test_proxy_logging.py \
test/unit/common/ring \ test/unit/common/ring \
test/unit/common/test_bufferedhttp.py \
test/unit/common/test_constraints.py \ test/unit/common/test_constraints.py \
test/unit/common/test_daemon.py \ test/unit/common/test_daemon.py \
test/unit/common/test_exceptions.py \ test/unit/common/test_exceptions.py \