From a2ff9eaeed983ff3bacf60ce32c6bc1e95cff5b5 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Thu, 9 Jan 2014 11:05:22 +0100 Subject: [PATCH] Port to python requests Since python requests has builtins SSL verification we can get ride of our own implementation. Increase tests coverage of heatclient.common.http along the way. Partial Implements: blueprint python-requests-port Change-Id: I04a169da2334acc91e538ca02cba79d9765752b5 --- heatclient/common/http.py | 247 +++++--------- heatclient/exc.py | 9 +- heatclient/tests/fakes.py | 25 +- heatclient/tests/test_common_http.py | 472 ++++++++++++++++----------- heatclient/tests/test_shell.py | 8 +- heatclient/v1/actions.py | 4 +- heatclient/v1/stacks.py | 6 +- requirements.txt | 1 + 8 files changed, 404 insertions(+), 368 deletions(-) diff --git a/heatclient/common/http.py b/heatclient/common/http.py index 0c0c0674..1843933e 100644 --- a/heatclient/common/http.py +++ b/heatclient/common/http.py @@ -16,23 +16,13 @@ import copy import logging import os -import posixpath -import requests import socket -from heatclient.openstack.common import jsonutils -from heatclient.openstack.common.py3kcompat import urlutils -from six.moves import http_client as httplib - -try: - import ssl -except ImportError: - #TODO(bcwaldon): Handle this failure more gracefully - pass - +import requests from heatclient import exc - +from heatclient.openstack.common import jsonutils +from heatclient.openstack.common.py3kcompat import urlutils LOG = logging.getLogger(__name__) if not LOG.handlers: @@ -41,6 +31,20 @@ USER_AGENT = 'python-heatclient' CHUNKSIZE = 1024 * 64 # 64kB +def get_system_ca_file(): + """Return path to system default CA file.""" + # Standard CA file locations for Debian/Ubuntu, RedHat/Fedora, + # Suse, FreeBSD/OpenBSD + ca_path = ['/etc/ssl/certs/ca-certificates.crt', + '/etc/pki/tls/certs/ca-bundle.crt', + '/etc/ssl/ca-bundle.pem', + '/etc/ssl/cert.pem'] + for ca in ca_path: + if os.path.exists(ca): + return ca + return None + + class HTTPClient(object): def __init__(self, endpoint, **kwargs): @@ -51,36 +55,24 @@ class HTTPClient(object): self.password = kwargs.get('password') self.region_name = kwargs.get('region_name') self.include_pass = kwargs.get('include_pass') - self.connection_params = self.get_connection_params(endpoint, **kwargs) + self.endpoint_url = endpoint - @staticmethod - def get_connection_params(endpoint, **kwargs): - parts = urlutils.urlparse(endpoint) + self.cert_file = kwargs.get('cert_file') + self.key_file = kwargs.get('key_file') - _args = (parts.hostname, parts.port, parts.path) - _kwargs = {'timeout': float(kwargs.get('timeout', 600))} + self.ssl_connection_params = { + 'ca_file': kwargs.get('ca_file'), + 'cert_file': kwargs.get('cert_file'), + 'key_file': kwargs.get('key_file'), + 'insecure': kwargs.get('insecure'), + } - if parts.scheme == 'https': - _class = VerifiedHTTPSConnection - _kwargs['ca_file'] = kwargs.get('ca_file', None) - _kwargs['cert_file'] = kwargs.get('cert_file', None) - _kwargs['key_file'] = kwargs.get('key_file', None) - _kwargs['insecure'] = kwargs.get('insecure', False) - elif parts.scheme == 'http': - _class = httplib.HTTPConnection - else: - msg = 'Unsupported scheme: %s' % parts.scheme - raise exc.InvalidEndpoint(msg) - - return (_class, _args, _kwargs) - - def get_connection(self): - _class = self.connection_params[0] - try: - return _class(*self.connection_params[1][0:2], - **self.connection_params[2]) - except httplib.InvalidURL: - raise exc.InvalidEndpoint() + self.verify_cert = None + if urlutils.urlparse(endpoint).scheme == "https": + if kwargs.get('insecure'): + self.verify_cert = False + else: + self.verify_cert = kwargs.get('ca_file', get_system_ca_file()) def log_curl_request(self, method, url, kwargs): curl = ['curl -i -X %s' % method] @@ -95,34 +87,34 @@ class HTTPClient(object): ('ca_file', '--cacert %s'), ] for (key, fmt) in conn_params_fmt: - value = self.connection_params[2].get(key) + value = self.ssl_connection_params.get(key) if value: curl.append(fmt % value) - if self.connection_params[2].get('insecure'): + if self.ssl_connection_params.get('insecure'): curl.append('-k') - if 'body' in kwargs: - curl.append('-d \'%s\'' % kwargs['body']) + if 'data' in kwargs: + curl.append('-d \'%s\'' % kwargs['data']) curl.append('%s%s' % (self.endpoint, url)) LOG.debug(' '.join(curl)) @staticmethod - def log_http_response(resp, body=None): - status = (resp.version / 10.0, resp.status, resp.reason) + def log_http_response(resp): + status = (resp.raw.version / 10.0, resp.status_code, resp.reason) dump = ['\nHTTP/%.1f %s %s' % status] - dump.extend(['%s: %s' % (k, v) for k, v in resp.getheaders()]) + dump.extend(['%s: %s' % (k, v) for k, v in resp.headers.items()]) dump.append('') - if body: - dump.extend([body, '']) + if resp.content: + dump.extend([resp.content.decode(), '']) LOG.debug('\n'.join(dump)) def _http_request(self, url, method, **kwargs): """Send an http request with the specified characteristics. - Wrapper around httplib.HTTP(S)Connection.request to handle tasks such - as setting headers and error handling. + Wrapper around requests.request to handle tasks such as + setting headers and error handling. """ # Copy the kwargs so we can reuse the original in case of redirects kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {})) @@ -139,16 +131,36 @@ class HTTPClient(object): kwargs['headers'].update(self.credentials_headers()) self.log_curl_request(method, url, kwargs) - conn = self.get_connection() + + if self.cert_file and self.key_file: + kwargs['cert'] = (self.cert_file, self.key_file) + + if self.verify_cert is not None: + kwargs['verify'] = self.verify_cert + + # We are not using requests builtin redirection on DELETE since it does + # not follow the RFC having to resend the same method on a + # redirect. For example if we do a DELETE on a URL and we get + # a 302 RFC says that we should follow that URL with the same + # method as before, requests doesn't follow that and send a + # GET instead for the method. See issue: + # https://github.com/kennethreitz/requests/issues/1704 + # hopefully this could be fixed as they say in a comment in a + # future point version i.e: 3.x + if method == 'DELETE': + allow_redirects = False + else: + allow_redirects = True try: - conn_params = self.connection_params[1][2] - conn_url = posixpath.normpath('%s/%s' % (conn_params, url)) - conn.request(method, conn_url, **kwargs) - resp = conn.getresponse() + resp = requests.request( + method, + self.endpoint_url + url, + allow_redirects=allow_redirects, + **kwargs) except socket.gaierror as e: message = ("Error finding address for %(url)s: %(e)s" % - {'url': url, 'e': e}) + {'url': self.endpoint_url + url, 'e': e}) raise exc.InvalidEndpoint(message=message) except (socket.error, socket.timeout) as e: endpoint = self.endpoint @@ -156,23 +168,21 @@ class HTTPClient(object): {'endpoint': endpoint, 'e': e}) raise exc.CommunicationError(message=message) - body_iter = ResponseBodyIterator(resp) - body_str = ''.join([chunk for chunk in body_iter]) - self.log_http_response(resp, body_str) + self.log_http_response(resp) if not 'X-Auth-Key' in kwargs['headers'] and \ - (resp.status == 401 or - (resp.status == 500 and "(HTTP 401)" in body_str)): + (resp.status_code == 401 or + (resp.status_code == 500 and "(HTTP 401)" in resp.content)): raise exc.HTTPUnauthorized("Authentication failed. Please try" " again with option " "--include-password or export " "HEAT_INCLUDE_PASSWORD=1\n%s" - % body_str) - elif 400 <= resp.status < 600: - raise exc.from_response(resp, body_str) - elif resp.status in (301, 302, 305): + % resp.content) + elif 400 <= resp.status_code < 600: + raise exc.from_response(resp) + elif resp.status_code in (301, 302, 305): # Redirected. Reissue the request to the new location. - location = resp.getheader('location', None) + location = resp.headers.get('location') if location is None: message = "Location not returned with 302" raise exc.InvalidEndpoint(message=message) @@ -183,10 +193,10 @@ class HTTPClient(object): message = "Prohibited endpoint redirect %s" % location raise exc.InvalidEndpoint(message=message) return self._http_request(location, method, **kwargs) - elif resp.status == 300: - raise exc.from_response(resp, body_str) + elif resp.status_code == 300: + raise exc.from_response(resp) - return resp, body_str + return resp def credentials_headers(self): creds = {} @@ -201,15 +211,14 @@ class HTTPClient(object): kwargs['headers'].setdefault('Content-Type', 'application/json') kwargs['headers'].setdefault('Accept', 'application/json') - if 'body' in kwargs: - kwargs['body'] = jsonutils.dumps(kwargs['body']) + if 'data' in kwargs: + kwargs['data'] = jsonutils.dumps(kwargs['data']) - resp, body_str = self._http_request(url, method, **kwargs) - - if 'application/json' in resp.getheader('content-type', None): - body = body_str + resp = self._http_request(url, method, **kwargs) + body = resp.content + if 'application/json' in resp.headers.get('content-type', ''): try: - body = jsonutils.loads(body) + body = resp.json() except ValueError: LOG.error('Could not decode response body as JSON') else: @@ -225,9 +234,7 @@ class HTTPClient(object): def client_request(self, method, url, **kwargs): resp, body = self.json_request(method, url, **kwargs) - r = requests.Response() - r._content = jsonutils.dumps(body) - return r + return resp def head(self, url, **kwargs): return self.client_request("HEAD", url, **kwargs) @@ -246,83 +253,3 @@ class HTTPClient(object): def patch(self, url, **kwargs): return self.client_request("PATCH", url, **kwargs) - - -class VerifiedHTTPSConnection(httplib.HTTPSConnection): - """httplib-compatibile connection using client-side SSL authentication - - :see http://code.activestate.com/recipes/ - 577548-https-httplib-client-connection-with-certificate-v/ - """ - - def __init__(self, host, port, key_file=None, cert_file=None, - ca_file=None, timeout=None, insecure=False): - httplib.HTTPSConnection.__init__(self, host, port, key_file=key_file, - cert_file=cert_file) - self.key_file = key_file - self.cert_file = cert_file - if ca_file is not None: - self.ca_file = ca_file - else: - self.ca_file = self.get_system_ca_file() - self.timeout = timeout - self.insecure = insecure - - def connect(self): - """Connect to a host on a given (SSL) port. - If ca_file is pointing somewhere, use it to check Server Certificate. - - Redefined/copied and extended from httplib.py:1105 (Python 2.6.x). - This is needed to pass cert_reqs=ssl.CERT_REQUIRED as parameter to - ssl.wrap_socket(), which forces SSL to check server certificate against - our client certificate. - """ - sock = socket.create_connection((self.host, self.port), self.timeout) - - if self._tunnel_host: - self.sock = sock - self._tunnel() - - if self.insecure is True: - kwargs = {'cert_reqs': ssl.CERT_NONE} - else: - kwargs = {'cert_reqs': ssl.CERT_REQUIRED, 'ca_certs': self.ca_file} - - if self.cert_file: - kwargs['certfile'] = self.cert_file - if self.key_file: - kwargs['keyfile'] = self.key_file - - self.sock = ssl.wrap_socket(sock, **kwargs) - - @staticmethod - def get_system_ca_file(): - """Return path to system default CA file.""" - # Standard CA file locations for Debian/Ubuntu, RedHat/Fedora, - # Suse, FreeBSD/OpenBSD - ca_path = ['/etc/ssl/certs/ca-certificates.crt', - '/etc/pki/tls/certs/ca-bundle.crt', - '/etc/ssl/ca-bundle.pem', - '/etc/ssl/cert.pem'] - for ca in ca_path: - if os.path.exists(ca): - return ca - return None - - -class ResponseBodyIterator(object): - """A class that acts as an iterator over an HTTP response.""" - - def __init__(self, resp): - self.resp = resp - - def __iter__(self): - while True: - yield self.next() - - def next(self): - chunk = self.resp.read(CHUNKSIZE) - if chunk: - return chunk - else: - raise StopIteration() diff --git a/heatclient/exc.py b/heatclient/exc.py index c3d41171..fd2f334b 100644 --- a/heatclient/exc.py +++ b/heatclient/exc.py @@ -169,11 +169,10 @@ for obj_name in dir(sys.modules[__name__]): _code_map[obj.code] = obj -def from_response(response, body_iter): - """Return an instance of an HTTPException based on httplib response.""" - cls = _code_map.get(response.status, HTTPException) - body_str = ''.join([chunk for chunk in body_iter]) - return cls(body_str) +def from_response(response): + """Return an instance of an HTTPException based on requests response.""" + cls = _code_map.get(response.status_code, HTTPException) + return cls(response.content) class NoTokenLookupException(Exception): diff --git a/heatclient/tests/fakes.py b/heatclient/tests/fakes.py index fe6c91df..5e8c4416 100644 --- a/heatclient/tests/fakes.py +++ b/heatclient/tests/fakes.py @@ -73,7 +73,7 @@ def script_heat_normal_error(): {'content-type': 'application/json'}, jsonutils.dumps(resp_dict)) http.HTTPClient.json_request('GET', '/stacks/bad').AndRaise( - exc.from_response(resp, jsonutils.dumps(resp_dict))) + exc.from_response(resp)) def script_heat_error(resp_string): @@ -82,7 +82,7 @@ def script_heat_error(resp_string): {'content-type': 'application/json'}, resp_string) http.HTTPClient.json_request('GET', '/stacks/bad').AndRaise( - exc.from_response(resp, resp_string)) + exc.from_response(resp)) def fake_headers(): @@ -104,15 +104,20 @@ class FakeKeystone(): self.auth_token = auth_token +class FakeRaw(): + version = 110 + + class FakeHTTPResponse(): version = 1.1 - def __init__(self, status, reason, headers, body): + def __init__(self, status_code, reason, headers, content): self.headers = headers - self.body = body - self.status = status + self.content = content + self.status_code = status_code self.reason = reason + self.raw = FakeRaw() def getheader(self, name, default=None): return self.headers.get(name, default) @@ -121,6 +126,12 @@ class FakeHTTPResponse(): return self.headers.items() def read(self, amt=None): - b = self.body - self.body = None + b = self.content + self.content = None return b + + def iter_content(self, chunksize): + return self.content + + def json(self): + return jsonutils.loads(self.content) diff --git a/heatclient/tests/test_common_http.py b/heatclient/tests/test_common_http.py index e153cd2c..3388c94d 100644 --- a/heatclient/tests/test_common_http.py +++ b/heatclient/tests/test_common_http.py @@ -10,7 +10,11 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging +import os +import socket +import requests import testtools from heatclient.common import http @@ -25,19 +29,19 @@ class HttpClientTest(testtools.TestCase): def setUp(self): super(HttpClientTest, self).setUp() self.m = mox.Mox() - self.m.StubOutClassWithMocks(http.httplib, 'HTTPConnection') - self.m.StubOutClassWithMocks(http.httplib, 'HTTPSConnection') + self.m.StubOutWithMock(requests, 'request') self.addCleanup(self.m.UnsetStubs) self.addCleanup(self.m.ResetAll) def test_http_raw_request(self): + headers = {'Content-Type': 'application/octet-stream', + 'User-Agent': 'python-heatclient'} + # Record a 200 - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/octet-stream', - 'User-Agent': 'python-heatclient'}) - mock_conn.getresponse().AndReturn( + mock_conn = http.requests.request('GET', 'http://example.com:8004', + allow_redirects=True, + headers=headers) + mock_conn.AndReturn( fakes.FakeHTTPResponse( 200, 'OK', {'content-type': 'application/octet-stream'}, @@ -45,9 +49,9 @@ class HttpClientTest(testtools.TestCase): # Replay, create client, assert self.m.ReplayAll() client = http.HTTPClient('http://example.com:8004') - resp, body = client.raw_request('GET', '') - self.assertEqual(200, resp.status) - self.assertEqual('', ''.join([x for x in body])) + resp = client.raw_request('GET', '') + self.assertEqual(200, resp.status_code) + self.assertEqual('', ''.join([x for x in resp.content])) self.m.VerifyAll() def test_token_or_credentials(self): @@ -58,46 +62,46 @@ class HttpClientTest(testtools.TestCase): '') # no token or credentials - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/octet-stream', - 'User-Agent': 'python-heatclient'}) - mock_conn.getresponse().AndReturn(fake200) + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', + allow_redirects=True, + headers={'Content-Type': 'application/octet-stream', + 'User-Agent': 'python-heatclient'}) + mock_conn.AndReturn(fake200) # credentials - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/octet-stream', - 'User-Agent': 'python-heatclient', - 'X-Auth-Key': 'pass', - 'X-Auth-User': 'user'}) - mock_conn.getresponse().AndReturn(fake200) + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', + allow_redirects=True, + headers={'Content-Type': 'application/octet-stream', + 'User-Agent': 'python-heatclient', + 'X-Auth-Key': 'pass', + 'X-Auth-User': 'user'}) + mock_conn.AndReturn(fake200) # token suppresses credentials - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/octet-stream', - 'User-Agent': 'python-heatclient', - 'X-Auth-Token': 'abcd1234'}) - mock_conn.getresponse().AndReturn(fake200) + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', + allow_redirects=True, + headers={'Content-Type': 'application/octet-stream', + 'User-Agent': 'python-heatclient', + 'X-Auth-Token': 'abcd1234'}) + mock_conn.AndReturn(fake200) # Replay, create client, assert self.m.ReplayAll() client = http.HTTPClient('http://example.com:8004') - resp, body = client.raw_request('GET', '') - self.assertEqual(200, resp.status) + resp = client.raw_request('GET', '') + self.assertEqual(200, resp.status_code) client.username = 'user' client.password = 'pass' - resp, body = client.raw_request('GET', '') - self.assertEqual(200, resp.status) + resp = client.raw_request('GET', '') + self.assertEqual(200, resp.status_code) client.auth_token = 'abcd1234' - resp, body = client.raw_request('GET', '') - self.assertEqual(200, resp.status) + resp = client.raw_request('GET', '') + self.assertEqual(200, resp.status_code) self.m.VerifyAll() def test_include_pass(self): @@ -108,49 +112,49 @@ class HttpClientTest(testtools.TestCase): '') # no token or credentials - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/octet-stream', - 'User-Agent': 'python-heatclient'}) - mock_conn.getresponse().AndReturn(fake200) + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', + allow_redirects=True, + headers={'Content-Type': 'application/octet-stream', + 'User-Agent': 'python-heatclient'}) + mock_conn.AndReturn(fake200) # credentials - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/octet-stream', - 'User-Agent': 'python-heatclient', - 'X-Auth-Key': 'pass', - 'X-Auth-User': 'user'}) - mock_conn.getresponse().AndReturn(fake200) + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', + allow_redirects=True, + headers={'Content-Type': 'application/octet-stream', + 'User-Agent': 'python-heatclient', + 'X-Auth-Key': 'pass', + 'X-Auth-User': 'user'}) + mock_conn.AndReturn(fake200) # token suppresses credentials - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/octet-stream', - 'User-Agent': 'python-heatclient', - 'X-Auth-Token': 'abcd1234', - 'X-Auth-Key': 'pass', - 'X-Auth-User': 'user'}) - mock_conn.getresponse().AndReturn(fake200) + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', + allow_redirects=True, + headers={'Content-Type': 'application/octet-stream', + 'User-Agent': 'python-heatclient', + 'X-Auth-Token': 'abcd1234', + 'X-Auth-Key': 'pass', + 'X-Auth-User': 'user'}) + mock_conn.AndReturn(fake200) # Replay, create client, assert self.m.ReplayAll() client = http.HTTPClient('http://example.com:8004') - resp, body = client.raw_request('GET', '') - self.assertEqual(200, resp.status) + resp = client.raw_request('GET', '') + self.assertEqual(200, resp.status_code) client.username = 'user' client.password = 'pass' client.include_pass = True - resp, body = client.raw_request('GET', '') - self.assertEqual(200, resp.status) + resp = client.raw_request('GET', '') + self.assertEqual(200, resp.status_code) client.auth_token = 'abcd1234' - resp, body = client.raw_request('GET', '') - self.assertEqual(200, resp.status) + resp = client.raw_request('GET', '') + self.assertEqual(200, resp.status_code) self.m.VerifyAll() def test_not_include_pass(self): @@ -161,12 +165,12 @@ class HttpClientTest(testtools.TestCase): '(HTTP 401)') # no token or credentials - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/octet-stream', - 'User-Agent': 'python-heatclient'}) - mock_conn.getresponse().AndReturn(fake500) + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', + allow_redirects=True, + headers={'Content-Type': 'application/octet-stream', + 'User-Agent': 'python-heatclient'}) + mock_conn.AndReturn(fake500) # Replay, create client, assert self.m.ReplayAll() @@ -183,31 +187,31 @@ class HttpClientTest(testtools.TestCase): '') # Specify region name - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/octet-stream', - 'X-Region-Name': 'RegionOne', - 'User-Agent': 'python-heatclient'}) - mock_conn.getresponse().AndReturn(fake200) + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', + allow_redirects=True, + headers={'Content-Type': 'application/octet-stream', + 'X-Region-Name': 'RegionOne', + 'User-Agent': 'python-heatclient'}) + mock_conn.AndReturn(fake200) # Replay, create client, assert self.m.ReplayAll() client = http.HTTPClient('http://example.com:8004') client.region_name = 'RegionOne' - resp, body = client.raw_request('GET', '') - self.assertEqual(200, resp.status) + resp = client.raw_request('GET', '') + self.assertEqual(200, resp.status_code) self.m.VerifyAll() def test_http_json_request(self): # Record a 200 - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/json', - 'Accept': 'application/json', - 'User-Agent': 'python-heatclient'}) - mock_conn.getresponse().AndReturn( + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', + allow_redirects=True, + headers={'Content-Type': 'application/json', + 'Accept': 'application/json', + 'User-Agent': 'python-heatclient'}) + mock_conn.AndReturn( fakes.FakeHTTPResponse( 200, 'OK', {'content-type': 'application/json'}, @@ -216,19 +220,20 @@ class HttpClientTest(testtools.TestCase): self.m.ReplayAll() client = http.HTTPClient('http://example.com:8004') resp, body = client.json_request('GET', '') - self.assertEqual(200, resp.status) + self.assertEqual(200, resp.status_code) self.assertEqual({}, body) self.m.VerifyAll() def test_http_json_request_w_req_body(self): # Record a 200 - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', body='"test-body"', - headers={'Content-Type': 'application/json', - 'Accept': 'application/json', - 'User-Agent': 'python-heatclient'}) - mock_conn.getresponse().AndReturn( + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', + body='test-body', + allow_redirects=True, + headers={'Content-Type': 'application/json', + 'Accept': 'application/json', + 'User-Agent': 'python-heatclient'}) + mock_conn.AndReturn( fakes.FakeHTTPResponse( 200, 'OK', {'content-type': 'application/json'}, @@ -237,19 +242,19 @@ class HttpClientTest(testtools.TestCase): self.m.ReplayAll() client = http.HTTPClient('http://example.com:8004') resp, body = client.json_request('GET', '', body='test-body') - self.assertEqual(200, resp.status) + self.assertEqual(200, resp.status_code) self.assertEqual({}, body) self.m.VerifyAll() def test_http_json_request_non_json_resp_cont_type(self): # Record a 200 - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', body='"test-body"', - headers={'Content-Type': 'application/json', - 'Accept': 'application/json', - 'User-Agent': 'python-heatclient'}) - mock_conn.getresponse().AndReturn( + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', body='test-body', + allow_redirects=True, + headers={'Content-Type': 'application/json', + 'Accept': 'application/json', + 'User-Agent': 'python-heatclient'}) + mock_conn.AndReturn( fakes.FakeHTTPResponse( 200, 'OK', {'content-type': 'not/json'}, @@ -258,19 +263,19 @@ class HttpClientTest(testtools.TestCase): self.m.ReplayAll() client = http.HTTPClient('http://example.com:8004') resp, body = client.json_request('GET', '', body='test-body') - self.assertEqual(200, resp.status) + self.assertEqual(200, resp.status_code) self.assertIsNone(body) self.m.VerifyAll() def test_http_json_request_invalid_json(self): # Record a 200 - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/json', - 'Accept': 'application/json', - 'User-Agent': 'python-heatclient'}) - mock_conn.getresponse().AndReturn( + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', + allow_redirects=True, + headers={'Content-Type': 'application/json', + 'Accept': 'application/json', + 'User-Agent': 'python-heatclient'}) + mock_conn.AndReturn( fakes.FakeHTTPResponse( 200, 'OK', {'content-type': 'application/json'}, @@ -279,31 +284,81 @@ class HttpClientTest(testtools.TestCase): self.m.ReplayAll() client = http.HTTPClient('http://example.com:8004') resp, body = client.json_request('GET', '') - self.assertEqual(200, resp.status) + self.assertEqual(200, resp.status_code) self.assertEqual('invalid-json', body) self.m.VerifyAll() + def test_http_manual_redirect_delete(self): + mock_conn = http.requests.request( + 'DELETE', 'http://example.com:8004/foo', + allow_redirects=False, + headers={'Content-Type': 'application/json', + 'Accept': 'application/json', + 'User-Agent': 'python-heatclient'}) + mock_conn.AndReturn( + fakes.FakeHTTPResponse( + 302, 'Found', + {'location': 'http://example.com:8004/foo/bar'}, + '')) + mock_conn = http.requests.request( + 'DELETE', 'http://example.com:8004/foo/bar', + allow_redirects=False, + headers={'Content-Type': 'application/json', + 'Accept': 'application/json', + 'User-Agent': 'python-heatclient'}) + mock_conn.AndReturn( + fakes.FakeHTTPResponse( + 200, 'OK', + {'content-type': 'application/json'}, + '{}')) + + self.m.ReplayAll() + + client = http.HTTPClient('http://example.com:8004/foo') + resp, body = client.json_request('DELETE', '') + + self.assertEqual(200, resp.status_code) + self.m.VerifyAll() + + def test_http_manual_redirect_prohibited(self): + mock_conn = http.requests.request( + 'DELETE', 'http://example.com:8004/foo', + allow_redirects=False, + headers={'Content-Type': 'application/json', + 'Accept': 'application/json', + 'User-Agent': 'python-heatclient'}) + mock_conn.AndReturn( + fakes.FakeHTTPResponse( + 302, 'Found', + {'location': 'http://example.com:8004/'}, + '')) + self.m.ReplayAll() + client = http.HTTPClient('http://example.com:8004/foo') + self.assertRaises(exc.InvalidEndpoint, + client.json_request, 'DELETE', '') + self.m.VerifyAll() + def test_http_json_request_redirect(self): # Record the 302 - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/json', - 'Accept': 'application/json', - 'User-Agent': 'python-heatclient'}) - mock_conn.getresponse().AndReturn( + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', + allow_redirects=True, + headers={'Content-Type': 'application/json', + 'Accept': 'application/json', + 'User-Agent': 'python-heatclient'}) + mock_conn.AndReturn( fakes.FakeHTTPResponse( 302, 'Found', {'location': 'http://example.com:8004'}, '')) # Record the following 200 - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/json', - 'Accept': 'application/json', - 'User-Agent': 'python-heatclient'}) - mock_conn.getresponse().AndReturn( + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', + allow_redirects=True, + headers={'Content-Type': 'application/json', + 'Accept': 'application/json', + 'User-Agent': 'python-heatclient'}) + mock_conn.AndReturn( fakes.FakeHTTPResponse( 200, 'OK', {'content-type': 'application/json'}, @@ -312,38 +367,19 @@ class HttpClientTest(testtools.TestCase): self.m.ReplayAll() client = http.HTTPClient('http://example.com:8004') resp, body = client.json_request('GET', '') - self.assertEqual(200, resp.status) - self.assertEqual({}, body) - self.m.VerifyAll() - - def test_http_json_request_prohibited_redirect(self): - # Record the 302 - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/json', - 'Accept': 'application/json', - 'User-Agent': 'python-heatclient'}) - mock_conn.getresponse().AndReturn( - fakes.FakeHTTPResponse( - 302, 'Found', - {'location': 'http://prohibited.example.com:8004'}, - '')) - # Replay, create client, assert - self.m.ReplayAll() - client = http.HTTPClient('http://example.com:8004') - self.assertRaises(exc.InvalidEndpoint, client.json_request, 'GET', '') + self.assertEqual(resp.status_code, 200) + self.assertEqual(body, {}) self.m.VerifyAll() def test_http_404_json_request(self): # Record a 404 - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/json', - 'Accept': 'application/json', - 'User-Agent': 'python-heatclient'}) - mock_conn.getresponse().AndReturn( + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', + allow_redirects=True, + headers={'Content-Type': 'application/json', + 'Accept': 'application/json', + 'User-Agent': 'python-heatclient'}) + mock_conn.AndReturn( fakes.FakeHTTPResponse( 404, 'OK', {'content-type': 'application/json'}, '{}')) @@ -360,13 +396,13 @@ class HttpClientTest(testtools.TestCase): def test_http_300_json_request(self): # Record a 300 - mock_conn = http.httplib.HTTPConnection('example.com', 8004, - timeout=600.0) - mock_conn.request('GET', '/', - headers={'Content-Type': 'application/json', - 'Accept': 'application/json', - 'User-Agent': 'python-heatclient'}) - mock_conn.getresponse().AndReturn( + mock_conn = http.requests.request( + 'GET', 'http://example.com:8004', + allow_redirects=True, + headers={'Content-Type': 'application/json', + 'Accept': 'application/json', + 'User-Agent': 'python-heatclient'}) + mock_conn.AndReturn( fakes.FakeHTTPResponse( 300, 'OK', {'content-type': 'application/json'}, '{}')) @@ -381,28 +417,90 @@ class HttpClientTest(testtools.TestCase): self.assertIsNotNone(e.message) self.m.VerifyAll() - #def test_https_json_request(self): - # # Record a 200 - # mock_conn = http.httplib.HTTPSConnection('example.com', 8004, - # '', timeout=600.0) - # mock_conn.request('GET', '/', - # headers={'Content-Type': 'application/json', - # 'Accept': 'application/json', - # 'User-Agent': 'python-heatclient'}) - # mock_conn.getresponse().AndReturn(fakes.FakeHTTPResponse(200, 'OK', - # {'content-type': 'application/json'}, - # '{}')) - # # Replay, create client, assert - # self.m.ReplayAll() - # client = http.HTTPClient('https://example.com:8004', - # ca_file='dummy', - # cert_file='dummy', - # key_file='dummy') - # resp, body = client.json_request('GET', '') - # self.assertEqual(200, resp.status) - # self.assertEqual({}, body) - # self.m.VerifyAll() - def test_fake_json_request(self): - self.assertRaises(exc.InvalidEndpoint, http.HTTPClient, - 'fake://example.com:8004') + headers = {'User-Agent': 'python-heatclient'} + mock_conn = http.requests.request('GET', 'fake://example.com:8004/', + allow_redirects=True, + headers=headers) + mock_conn.AndRaise(socket.gaierror) + self.m.ReplayAll() + + client = http.HTTPClient('fake://example.com:8004') + self.assertRaises(exc.InvalidEndpoint, + client._http_request, "/", "GET") + self.m.VerifyAll() + + def test_debug_curl_command(self): + self.m.StubOutWithMock(logging.Logger, 'debug') + + ssl_connection_params = {'ca_file': 'TEST_CA', + 'cert_file': 'TEST_CERT', + 'key_file': 'TEST_KEY', + 'insecure': 'TEST_NSA'} + + headers = {'key': 'value'} + + mock_logging_debug = logging.Logger.debug( + "curl -i -X GET -H 'key: value' --key TEST_KEY " + "--cert TEST_CERT --cacert TEST_CA " + "-k http://foo/bar" + ) + mock_logging_debug.AndReturn(None) + self.m.ReplayAll() + + client = http.HTTPClient('http://foo') + client.ssl_connection_params = ssl_connection_params + client.log_curl_request('GET', '/bar', {'headers': headers}) + + self.m.VerifyAll() + + def test_http_request_socket_error(self): + headers = {'User-Agent': 'python-heatclient'} + mock_conn = http.requests.request('GET', 'http://example.com:8004/', + allow_redirects=True, + headers=headers) + mock_conn.AndRaise(socket.error) + self.m.ReplayAll() + + client = http.HTTPClient('http://example.com:8004') + self.assertRaises(exc.CommunicationError, + client._http_request, "/", "GET") + self.m.VerifyAll() + + def test_http_request_socket_timeout(self): + headers = {'User-Agent': 'python-heatclient'} + mock_conn = http.requests.request('GET', 'http://example.com:8004/', + allow_redirects=True, + headers=headers) + mock_conn.AndRaise(socket.timeout) + self.m.ReplayAll() + + client = http.HTTPClient('http://example.com:8004') + self.assertRaises(exc.CommunicationError, + client._http_request, "/", "GET") + self.m.VerifyAll() + + def test_get_system_ca_file(self): + chosen = '/etc/ssl/certs/ca-certificates.crt' + self.m.StubOutWithMock(os.path, 'exists') + os.path.exists(chosen).AndReturn(chosen) + self.m.ReplayAll() + + ca = http.get_system_ca_file() + self.assertEqual(ca, chosen) + + self.m.VerifyAll() + + def test_insecure_verify_cert_None(self): + client = http.HTTPClient('https://foo', insecure=True) + self.assertFalse(client.verify_cert) + + def test_passed_cert_to_verify_cert(self): + client = http.HTTPClient('https://foo', ca_file="NOWHERE") + self.assertEqual(client.verify_cert, "NOWHERE") + + self.m.StubOutWithMock(http, 'get_system_ca_file') + http.get_system_ca_file().AndReturn("SOMEWHERE") + self.m.ReplayAll() + client = http.HTTPClient('https://foo') + self.assertEqual(client.verify_cert, "SOMEWHERE") diff --git a/heatclient/tests/test_shell.py b/heatclient/tests/test_shell.py index c5aa49b8..0205d1ae 100644 --- a/heatclient/tests/test_shell.py +++ b/heatclient/tests/test_shell.py @@ -596,7 +596,7 @@ class ShellTestUserPass(ShellBase): {'location': 'http://no.where/v1/tenant_id/stacks/teststack2/2'}, None) http.HTTPClient.json_request( - 'POST', '/stacks', body=mox.IgnoreArg(), + 'POST', '/stacks', data=mox.IgnoreArg(), headers={'X-Auth-Key': 'password', 'X-Auth-User': 'username'} ).AndReturn((resp, None)) fakes.script_heat_list() @@ -634,7 +634,7 @@ class ShellTestUserPass(ShellBase): six.StringIO('{}')) http.HTTPClient.json_request( - 'POST', '/stacks', body=mox.IgnoreArg(), + 'POST', '/stacks', data=mox.IgnoreArg(), headers={'X-Auth-Key': 'password', 'X-Auth-User': 'username'} ).AndReturn((resp, None)) fakes.script_heat_list() @@ -673,7 +673,7 @@ class ShellTestUserPass(ShellBase): {'location': 'http://no.where/v1/tenant_id/stacks/teststack2/2'}, None) http.HTTPClient.json_request( - 'POST', '/stacks', body=mox.IgnoreArg(), + 'POST', '/stacks', data=mox.IgnoreArg(), headers={'X-Auth-Key': 'password', 'X-Auth-User': 'username'} ).AndReturn((resp, None)) @@ -706,7 +706,7 @@ class ShellTestUserPass(ShellBase): 'The request is accepted for processing.') http.HTTPClient.json_request( 'PUT', '/stacks/teststack2/2', - body=mox.IgnoreArg(), + data=mox.IgnoreArg(), headers={'X-Auth-Key': 'password', 'X-Auth-User': 'username'} ).AndReturn((resp, None)) fakes.script_heat_list() diff --git a/heatclient/v1/actions.py b/heatclient/v1/actions.py index e615912f..afae9a6c 100644 --- a/heatclient/v1/actions.py +++ b/heatclient/v1/actions.py @@ -38,11 +38,11 @@ class ActionManager(stacks.StackChildManager): body = {'suspend': None} resp, body = self.client.json_request('POST', '/stacks/%s/actions' % stack_id, - body=body) + data=body) def resume(self, stack_id): """Resume a stack.""" body = {'resume': None} resp, body = self.client.json_request('POST', '/stacks/%s/actions' % stack_id, - body=body) + data=body) diff --git a/heatclient/v1/stacks.py b/heatclient/v1/stacks.py index e1843641..4fe215a5 100644 --- a/heatclient/v1/stacks.py +++ b/heatclient/v1/stacks.py @@ -105,14 +105,14 @@ class StackManager(base.BaseManager): """Create a stack.""" headers = self.client.credentials_headers() resp, body = self.client.json_request('POST', '/stacks', - body=kwargs, headers=headers) + data=kwargs, headers=headers) return body def update(self, stack_id, **kwargs): """Update a stack.""" headers = self.client.credentials_headers() resp, body = self.client.json_request('PUT', '/stacks/%s' % stack_id, - body=kwargs, headers=headers) + data=kwargs, headers=headers) def delete(self, stack_id): """Delete a stack.""" @@ -138,7 +138,7 @@ class StackManager(base.BaseManager): def validate(self, **kwargs): """Validate a stack template.""" - resp, body = self.client.json_request('POST', '/validate', body=kwargs) + resp, body = self.client.json_request('POST', '/validate', data=kwargs) return body diff --git a/requirements.txt b/requirements.txt index cc8df340..900942ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ PrettyTable>=0.7,<0.8 python-keystoneclient>=0.4.2 PyYAML>=3.1.0 six>=1.4.1 +requests>=1.1