diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index aee9c192c0..00f7e3f87e 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -109,6 +109,15 @@ use = egg:swift#proxy # set log_level = INFO # set log_address = /dev/log # +# When deployed behind a proxy, load balancer, or SSL terminator that is +# configured to speak the human-readable (v1) PROXY protocol (see +# http://www.haproxy.org/download/1.7/doc/proxy-protocol.txt), you should set +# this option to true. The proxy-server will populate the client connection +# information using the PROXY protocol and reject any connection missing a +# valid PROXY line with a 400. Only v1 (human-readable) of the PROXY protocol +# is supported. +# require_proxy_protocol = false +# # log_handoffs = true # recheck_account_existence = 60 # recheck_container_existence = 60 diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 2a9409d92e..9577230f7a 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -420,18 +420,99 @@ def load_app_config(conf_file): return app_conf +class SwiftHttpProtocol(wsgi.HttpProtocol): + default_request_version = "HTTP/1.0" + + def log_request(self, *a): + """ + Turn off logging requests by the underlying WSGI software. + """ + pass + + def log_message(self, f, *a): + """ + Redirect logging other messages by the underlying WSGI software. + """ + logger = getattr(self.server.app, 'logger', None) or self.server.log + logger.error('ERROR WSGI: ' + f, *a) + + +class SwiftHttpProxiedProtocol(SwiftHttpProtocol): + """ + Protocol object that speaks HTTP, including multiple requests, but with + a single PROXY line as the very first thing coming in over the socket. + This is so we can learn what the client's IP address is when Swift is + behind a TLS terminator, like hitch, that does not understand HTTP and + so cannot add X-Forwarded-For or other similar headers. + + See http://www.haproxy.org/download/1.7/doc/proxy-protocol.txt for + protocol details. + """ + def handle_error(self, connection_line): + if not six.PY2: + connection_line = connection_line.decode('latin-1') + + # No further processing will proceed on this connection under any + # circumstances. We always send the request into the superclass to + # handle any cleanup - this ensures that the request will not be + # processed. + self.rfile.close() + # We don't really have any confidence that an HTTP Error will be + # processable by the client as our transmission broken down between + # ourselves and our gateway proxy before processing the client + # protocol request. Hopefully the operator will know what to do! + msg = 'Invalid PROXY line %r' % connection_line + self.log_message(msg) + # Even assuming HTTP we don't even known what version of HTTP the + # client is sending? This entire endeavor seems questionable. + self.request_version = self.default_request_version + # appease http.server + self.command = 'PROXY' + self.send_error(400, msg) + + def handle(self): + """Handle multiple requests if necessary.""" + # ensure the opening line for the connection is a valid PROXY protcol + # line; this is the only IO we do on this connection before any + # additional wrapping further pollutes the raw socket. + connection_line = self.rfile.readline(self.server.url_length_limit) + + if connection_line.startswith(b'PROXY'): + proxy_parts = connection_line.split(b' ') + if len(proxy_parts) >= 2 and proxy_parts[0] == b'PROXY': + if proxy_parts[1] in (b'TCP4', b'TCP6') and \ + len(proxy_parts) == 6: + if six.PY2: + self.client_address = (proxy_parts[2], proxy_parts[4]) + else: + self.client_address = ( + proxy_parts[2].decode('latin-1'), + proxy_parts[4].decode('latin-1')) + elif proxy_parts[1].startswith(b'UNKNOWN'): + # "UNKNOWN", in PROXY protocol version 1, means "not + # TCP4 or TCP6". This includes completely legitimate + # things like QUIC or Unix domain sockets. The PROXY + # protocol (section 2.1) states that the receiver + # (that's us) MUST ignore anything after "UNKNOWN" and + # before the CRLF, essentially discarding the first + # line. + pass + else: + self.handle_error(connection_line) + else: + self.handle_error(connection_line) + else: + self.handle_error(connection_line) + + return SwiftHttpProtocol.handle(self) + + def run_server(conf, logger, sock, global_conf=None): # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on # some platforms. This locks in reported times to UTC. os.environ['TZ'] = 'UTC+0' time.tzset() - wsgi.HttpProtocol.default_request_version = "HTTP/1.0" - # Turn off logging requests by the underlying WSGI software. - wsgi.HttpProtocol.log_request = lambda *a: None - # Redirect logging other messages by the underlying WSGI software. - wsgi.HttpProtocol.log_message = \ - lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a) wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60) eventlet.hubs.use_hub(get_hub()) @@ -451,15 +532,25 @@ def run_server(conf, logger, sock, global_conf=None): app = loadapp(conf['__file__'], global_conf=global_conf) max_clients = int(conf.get('max_clients', '1024')) pool = RestrictedGreenPool(size=max_clients) + + # Select which protocol class to use (normal or one expecting PROXY + # protocol) + if config_true_value(conf.get('require_proxy_protocol', 'no')): + protocol_class = SwiftHttpProxiedProtocol + else: + protocol_class = SwiftHttpProtocol + + server_kwargs = { + 'custom_pool': pool, + 'protocol': protocol_class, + } + # Disable capitalizing headers in Eventlet if possible. This is + # necessary for the AWS SDK to work with swift3 middleware. + argspec = inspect.getargspec(wsgi.server) + if 'capitalize_response_headers' in argspec.args: + server_kwargs['capitalize_response_headers'] = False try: - # Disable capitalizing headers in Eventlet if possible. This is - # necessary for the AWS SDK to work with swift3 middleware. - argspec = inspect.getargspec(wsgi.server) - if 'capitalize_response_headers' in argspec.args: - wsgi.server(sock, app, wsgi_logger, custom_pool=pool, - capitalize_response_headers=False) - else: - wsgi.server(sock, app, wsgi_logger, custom_pool=pool) + wsgi.server(sock, app, wsgi_logger, **server_kwargs) except socket.error as err: if err[0] != errno.EINVAL: raise diff --git a/test/functional/__init__.py b/test/functional/__init__.py index 7f69a79037..dcbc064833 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -53,7 +53,8 @@ from test.unit import SkipTest from swift.common import constraints, utils, ring, storage_policy from swift.common.ring import Ring -from swift.common.wsgi import monkey_patch_mimetools, loadapp +from swift.common.wsgi import ( + monkey_patch_mimetools, loadapp, SwiftHttpProtocol) from swift.common.utils import config_true_value, split_path from swift.account import server as account_server from swift.container import server as container_server @@ -626,13 +627,6 @@ def in_process_setup(the_object_server=object_server): 'port': con2lis.getsockname()[1]}], 30), f) - eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0" - # Turn off logging requests by the underlying WSGI software. - eventlet.wsgi.HttpProtocol.log_request = lambda *a: None - logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi') - # Redirect logging other messages by the underlying WSGI software. - eventlet.wsgi.HttpProtocol.log_message = \ - lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a) # Default to only 4 seconds for in-process functional test runs eventlet.wsgi.WRITE_TIMEOUT = 4 @@ -659,7 +653,9 @@ def in_process_setup(the_object_server=object_server): ] if show_debug_logs: - logger = debug_logger('proxy') + logger = get_logger_name('proxy') + else: + logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi') def get_logger(name, *args, **kwargs): return logger @@ -675,13 +671,19 @@ def in_process_setup(the_object_server=object_server): nl = utils.NullLogger() global proxy_srv proxy_srv = prolis - prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl) - acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl) - acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl) - con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl) - con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl) + prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl, + protocol=SwiftHttpProtocol) + acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl, + protocol=SwiftHttpProtocol) + acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl, + protocol=SwiftHttpProtocol) + con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl, + protocol=SwiftHttpProtocol) + con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl, + protocol=SwiftHttpProtocol) - objspa = [eventlet.spawn(eventlet.wsgi.server, objsrv[0], objsrv[1], nl) + objspa = [eventlet.spawn(eventlet.wsgi.server, objsrv[0], objsrv[1], nl, + protocol=SwiftHttpProtocol) for objsrv in objsrvs] global _test_coros diff --git a/test/unit/common/test_wsgi.py b/test/unit/common/test_wsgi.py index 8e88d09b59..5064b5db50 100644 --- a/test/unit/common/test_wsgi.py +++ b/test/unit/common/test_wsgi.py @@ -15,6 +15,7 @@ """Tests for swift.common.wsgi""" +from argparse import Namespace import errno import logging import socket @@ -22,6 +23,9 @@ import unittest import os from textwrap import dedent from collections import defaultdict +import types + +import eventlet.wsgi import six from six import BytesIO @@ -480,8 +484,6 @@ class TestWSGI(unittest.TestCase): logger = logging.getLogger('test') sock = listen_zero() wsgi.run_server(conf, logger, sock) - self.assertEqual('HTTP/1.0', - _wsgi.HttpProtocol.default_request_version) self.assertEqual(30, _wsgi.WRITE_TIMEOUT) _wsgi_evt.hubs.use_hub.assert_called_with(utils.get_hub()) _wsgi_evt.debug.hub_exceptions.assert_called_with(False) @@ -495,6 +497,61 @@ class TestWSGI(unittest.TestCase): self.assertTrue('custom_pool' in kwargs) self.assertEqual(1000, kwargs['custom_pool'].size) + proto_class = kwargs['protocol'] + self.assertEqual(proto_class, wsgi.SwiftHttpProtocol) + self.assertEqual('HTTP/1.0', proto_class.default_request_version) + + def test_run_server_proxied(self): + config = """ + [DEFAULT] + client_timeout = 30 + max_clients = 1000 + swift_dir = TEMPDIR + + [pipeline:main] + pipeline = proxy-server + + [app:proxy-server] + use = egg:swift#proxy + # these "set" values override defaults + set client_timeout = 20 + set max_clients = 10 + require_proxy_protocol = true + """ + + contents = dedent(config) + with temptree(['proxy-server.conf']) as t: + conf_file = os.path.join(t, 'proxy-server.conf') + with open(conf_file, 'w') as f: + f.write(contents.replace('TEMPDIR', t)) + _fake_rings(t) + with mock.patch('swift.proxy.server.Application.' + 'modify_wsgi_pipeline'), \ + mock.patch('swift.common.wsgi.wsgi') as _wsgi, \ + mock.patch('swift.common.wsgi.eventlet') as _eventlet, \ + mock.patch('swift.common.wsgi.inspect'): + conf = wsgi.appconfig(conf_file, + name='proxy-server') + logger = logging.getLogger('test') + sock = listen_zero() + wsgi.run_server(conf, logger, sock) + self.assertEqual(20, _wsgi.WRITE_TIMEOUT) + _eventlet.hubs.use_hub.assert_called_with(utils.get_hub()) + _eventlet.debug.hub_exceptions.assert_called_with(False) + self.assertTrue(_wsgi.server.called) + args, kwargs = _wsgi.server.call_args + server_sock, server_app, server_logger = args + self.assertEqual(sock, server_sock) + self.assertTrue(isinstance(server_app, swift.proxy.server.Application)) + self.assertEqual(20, server_app.client_timeout) + self.assertTrue(isinstance(server_logger, wsgi.NullLogger)) + self.assertTrue('custom_pool' in kwargs) + self.assertEqual(10, kwargs['custom_pool'].size) + + proto_class = kwargs['protocol'] + self.assertEqual(proto_class, wsgi.SwiftHttpProxiedProtocol) + self.assertEqual('HTTP/1.0', proto_class.default_request_version) + def test_run_server_with_latest_eventlet(self): config = """ [DEFAULT] @@ -530,6 +587,9 @@ class TestWSGI(unittest.TestCase): self.assertTrue(_wsgi.server.called) args, kwargs = _wsgi.server.call_args self.assertEqual(kwargs.get('capitalize_response_headers'), False) + self.assertTrue('protocol' in kwargs) + self.assertEqual('HTTP/1.0', + kwargs['protocol'].default_request_version) def test_run_server_conf_dir(self): config_dir = { @@ -566,8 +626,6 @@ class TestWSGI(unittest.TestCase): wsgi.run_server(conf, logger, sock) self.assertTrue(os.environ['TZ'] is not '') - self.assertEqual('HTTP/1.0', - _wsgi.HttpProtocol.default_request_version) self.assertEqual(30, _wsgi.WRITE_TIMEOUT) _wsgi_evt.hubs.use_hub.assert_called_with(utils.get_hub()) _wsgi_evt.debug.hub_exceptions.assert_called_with(False) @@ -578,6 +636,9 @@ class TestWSGI(unittest.TestCase): self.assertTrue(isinstance(server_app, swift.proxy.server.Application)) self.assertTrue(isinstance(server_logger, wsgi.NullLogger)) self.assertTrue('custom_pool' in kwargs) + self.assertTrue('protocol' in kwargs) + self.assertEqual('HTTP/1.0', + kwargs['protocol'].default_request_version) def test_run_server_debug(self): config = """ @@ -615,8 +676,6 @@ class TestWSGI(unittest.TestCase): logger = logging.getLogger('test') sock = listen_zero() wsgi.run_server(conf, logger, sock) - self.assertEqual('HTTP/1.0', - _wsgi.HttpProtocol.default_request_version) self.assertEqual(30, _wsgi.WRITE_TIMEOUT) _wsgi_evt.hubs.use_hub.assert_called_with(utils.get_hub()) _wsgi_evt.debug.hub_exceptions.assert_called_with(True) @@ -629,6 +688,9 @@ class TestWSGI(unittest.TestCase): self.assertIsNone(server_logger) self.assertTrue('custom_pool' in kwargs) self.assertEqual(1000, kwargs['custom_pool'].size) + self.assertTrue('protocol' in kwargs) + self.assertEqual('HTTP/1.0', + kwargs['protocol'].default_request_version) def test_appconfig_dir_ignores_hidden_files(self): config_dir = { @@ -948,6 +1010,193 @@ class TestWSGI(unittest.TestCase): self.assertIs(newenv.get('swift.infocache'), oldenv['swift.infocache']) +class TestSwiftHttpProtocol(unittest.TestCase): + def setUp(self): + patcher = mock.patch('swift.common.wsgi.wsgi.HttpProtocol') + self.mock_super = patcher.start() + self.addCleanup(patcher.stop) + + def _proto_obj(self): + # Make an object we can exercise... note the base class's __init__() + # does a bunch of work, so we just new up an object like eventlet.wsgi + # does. + proto_class = wsgi.SwiftHttpProtocol + try: + the_obj = types.InstanceType(proto_class) + except AttributeError: + the_obj = proto_class.__new__(proto_class) + # Install some convenience mocks + the_obj.server = Namespace(app=Namespace(logger=mock.Mock()), + url_length_limit=777, + log=mock.Mock()) + the_obj.send_error = mock.Mock() + + return the_obj + + def test_swift_http_protocol_log_request(self): + proto_obj = self._proto_obj() + self.assertEqual(None, proto_obj.log_request('ignored')) + + def test_swift_http_protocol_log_message(self): + proto_obj = self._proto_obj() + + proto_obj.log_message('a%sc', 'b') + self.assertEqual([mock.call.error('ERROR WSGI: a%sc', 'b')], + proto_obj.server.app.logger.mock_calls) + + def test_swift_http_protocol_log_message_no_logger(self): + # If the app somehow had no logger attribute or it was None, don't blow + # up + proto_obj = self._proto_obj() + delattr(proto_obj.server.app, 'logger') + + proto_obj.log_message('a%sc', 'b') + self.assertEqual([mock.call.error('ERROR WSGI: a%sc', 'b')], + proto_obj.server.log.mock_calls) + + proto_obj.server.log.reset_mock() + proto_obj.server.app.logger = None + + proto_obj.log_message('a%sc', 'b') + self.assertEqual([mock.call.error('ERROR WSGI: a%sc', 'b')], + proto_obj.server.log.mock_calls) + + def test_swift_http_protocol_parse_request_no_proxy(self): + proto_obj = self._proto_obj() + proto_obj.raw_requestline = b'jimmy jam' + proto_obj.client_address = ('a', '123') + + self.assertEqual(False, proto_obj.parse_request()) + + self.assertEqual([], self.mock_super.mock_calls) + self.assertEqual([ + mock.call(400, "Bad HTTP/0.9 request type ('jimmy')"), + ], proto_obj.send_error.mock_calls) + self.assertEqual(('a', '123'), proto_obj.client_address) + + +class TestProxyProtocol(unittest.TestCase): + def _run_bytes_through_protocol(self, bytes_from_client, protocol_class): + rfile = BytesIO(bytes_from_client) + wfile = BytesIO() + + # All this fakery is needed to make the WSGI server process one + # connection, possibly with multiple requests, in the main + # greenthread. It doesn't hurt correctness if the function is called + # in a separate greenthread, but it makes using the debugger harder. + class FakeGreenthread(object): + def link(self, a_callable, *args): + a_callable(self, *args) + + class FakePool(object): + def spawn(self, a_callable, *args, **kwargs): + a_callable(*args, **kwargs) + return FakeGreenthread() + + def spawn_n(self, a_callable, *args, **kwargs): + a_callable(*args, **kwargs) + + def waitall(self): + pass + + def dinky_app(env, start_response): + start_response("200 OK", []) + body = "got addr: %s %s\r\n" % ( + env.get("REMOTE_ADDR", ""), + env.get("REMOTE_PORT", "")) + return [body.encode("utf-8")] + + fake_tcp_socket = mock.Mock( + setsockopt=lambda *a: None, + makefile=lambda mode, bufsize: rfile if 'r' in mode else wfile, + ) + fake_listen_socket = mock.Mock(accept=mock.MagicMock( + side_effect=[[fake_tcp_socket, ('127.0.0.1', 8359)], + # KeyboardInterrupt breaks the WSGI server out of + # its infinite accept-process-close loop. + KeyboardInterrupt])) + + # If we let the WSGI server close rfile/wfile then we can't access + # their contents any more. + with mock.patch.object(wfile, 'close', lambda: None), \ + mock.patch.object(rfile, 'close', lambda: None): + eventlet.wsgi.server( + fake_listen_socket, dinky_app, + protocol=protocol_class, + custom_pool=FakePool(), + log_output=False, # quiet the test run + ) + return wfile.getvalue() + + def test_request_with_proxy(self): + bytes_out = self._run_bytes_through_protocol(( + b"PROXY TCP4 192.168.0.1 192.168.0.11 56423 443\r\n" + b"GET /someurl HTTP/1.0\r\n" + b"User-Agent: something or other\r\n" + b"\r\n" + ), wsgi.SwiftHttpProxiedProtocol) + + lines = [l for l in bytes_out.split(b"\r\n") if l] + self.assertEqual(lines[0], b"HTTP/1.1 200 OK") # sanity check + self.assertEqual(lines[-1], b"got addr: 192.168.0.1 56423") + + def test_multiple_requests_with_proxy(self): + bytes_out = self._run_bytes_through_protocol(( + b"PROXY TCP4 192.168.0.1 192.168.0.11 56423 443\r\n" + b"GET /someurl HTTP/1.1\r\n" + b"User-Agent: something or other\r\n" + b"\r\n" + b"GET /otherurl HTTP/1.1\r\n" + b"User-Agent: something or other\r\n" + b"Connection: close\r\n" + b"\r\n" + ), wsgi.SwiftHttpProxiedProtocol) + + lines = bytes_out.split(b"\r\n") + self.assertEqual(lines[0], b"HTTP/1.1 200 OK") # sanity check + + # the address in the PROXY line is applied to every request + addr_lines = [l for l in lines if l.startswith(b"got addr")] + self.assertEqual(addr_lines, [b"got addr: 192.168.0.1 56423"] * 2) + + def test_missing_proxy_line(self): + bytes_out = self._run_bytes_through_protocol(( + # whoops, no PROXY line here + b"GET /someurl HTTP/1.0\r\n" + b"User-Agent: something or other\r\n" + b"\r\n" + ), wsgi.SwiftHttpProxiedProtocol) + + lines = [l for l in bytes_out.split(b"\r\n") if l] + self.assertIn(b"400 Invalid PROXY line", lines[0]) + + def test_malformed_proxy_lines(self): + for bad_line in [b'PROXY jojo', + b'PROXYjojo a b c d e', + b'PROXY a b c d e', # bad INET protocol and family + ]: + bytes_out = self._run_bytes_through_protocol( + bad_line, wsgi.SwiftHttpProxiedProtocol) + lines = [l for l in bytes_out.split(b"\r\n") if l] + self.assertIn(b"400 Invalid PROXY line", lines[0]) + + def test_unknown_client_addr(self): + # For "UNKNOWN", the rest of the line before the CRLF may be omitted by + # the sender, and the receiver must ignore anything presented before + # the CRLF is found. + for unknown_line in [b'PROXY UNKNOWN', # mimimal valid unknown + b'PROXY UNKNOWNblahblah', # also valid + b'PROXY UNKNOWN a b c d']: + bytes_out = self._run_bytes_through_protocol(( + unknown_line + (b"\r\n" + b"GET /someurl HTTP/1.0\r\n" + b"User-Agent: something or other\r\n" + b"\r\n") + ), wsgi.SwiftHttpProxiedProtocol) + lines = [l for l in bytes_out.split(b"\r\n") if l] + self.assertIn(b"200 OK", lines[0]) + + class TestServersPerPortStrategy(unittest.TestCase): def setUp(self): self.logger = FakeLogger()