openstack/neutron
Code Issues Proposed changes

Fixing the 500 HTTP code in the metadata service if Nova is down

Browse Source 
If the Nova metadata service is unavailable, the requests.request()
function may raise a ConnectionError. This results in the upper code
returning a 500 HTTP status code to the user along with a traceback.
Let's handle this scenario and instead return a 503 HTTP status code
(service unavailable).

If the Nova service is down and is behind another proxy (such as
Nginx), then instead of a ConnectionError, the request may result in
receiving a 502 or 503 HTTP status code. Let's also consider this
situation and add support for an additional 504 code.

Closes-Bug: #2059032
Change-Id: I16be18c46a6796224b0793dc385b0ddec01739c4
This commit is contained in:
Anton Kurbatov 2024-03-25 18:49:52 +00:00
parent 4e9d03d29f
commit 6395b4fe8e
5 changed files with 78 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
neutron/agent/metadata/agent.py
View File

@ -246,12 +246,18 @@ class MetadataProxyHandler(object):
client_cert = (self.conf.nova_client_cert,
self.conf.nova_client_priv_key)
try:
resp = requests.request(method=req.method, url=url,
headers=headers,
data=req.body,
cert=client_cert,
verify=verify_cert,
timeout=60)
except requests.ConnectionError:
msg = _('The remote metadata server is temporarily unavailable. '
'Please try again later.')
explanation = str(msg)
return webob.exc.HTTPServiceUnavailable(explanation=explanation)
if resp.status_code == 200:
req.response.content_type = resp.headers['content-type']
@ -264,12 +270,6 @@ class MetadataProxyHandler(object):
'response usually occurs when shared secrets do not match.'
)
return webob.exc.HTTPForbidden()
elif resp.status_code == 400:
return webob.exc.HTTPBadRequest()
elif resp.status_code == 404:
return webob.exc.HTTPNotFound()
elif resp.status_code == 409:
return webob.exc.HTTPConflict()
elif resp.status_code == 500:
msg = _(
'Remote metadata server experienced an internal server error.'
@ -277,6 +277,9 @@ class MetadataProxyHandler(object):
LOG.warning(msg)
explanation = str(msg)
return webob.exc.HTTPInternalServerError(explanation=explanation)
elif resp.status_code in (400, 404, 409, 502, 503, 504):
webob_exc_cls = webob.exc.status_map.get(resp.status_code)
return webob_exc_cls()
else:
raise Exception(_('Unexpected response code: %s') %
resp.status_code)

15
neutron/agent/ovn/metadata/server.py
View File

@ -168,12 +168,18 @@ class MetadataProxyHandler(object):
client_cert = (self.conf.nova_client_cert,
self.conf.nova_client_priv_key)
try:
resp = requests.request(method=req.method, url=url,
headers=headers,
data=req.body,
cert=client_cert,
verify=verify_cert,
timeout=60)
except requests.ConnectionError:
msg = _('The remote metadata server is temporarily unavailable. '
'Please try again later.')
explanation = str(msg)
return webob.exc.HTTPServiceUnavailable(explanation=explanation)
if resp.status_code == 200:
req.response.content_type = resp.headers['content-type']
@ -186,12 +192,6 @@ class MetadataProxyHandler(object):
'response usually occurs when shared secrets do not match.'
)
return webob.exc.HTTPForbidden()
elif resp.status_code == 400:
return webob.exc.HTTPBadRequest()
elif resp.status_code == 404:
return webob.exc.HTTPNotFound()
elif resp.status_code == 409:
return webob.exc.HTTPConflict()
elif resp.status_code == 500:
msg = _(
'Remote metadata server experienced an internal server error.'
@ -199,6 +199,9 @@ class MetadataProxyHandler(object):
LOG.warning(msg)
explanation = str(msg)
return webob.exc.HTTPInternalServerError(explanation=explanation)
elif resp.status_code in (400, 404, 409, 502, 503, 504):
webob_exc_cls = webob.exc.status_map.get(resp.status_code)
return webob_exc_cls()
else:
raise Exception(_('Unexpected response code: %s') %
resp.status_code)

21
neutron/tests/unit/agent/metadata/test_agent.py
View File

@ -17,6 +17,7 @@ from unittest import mock
import ddt
import netaddr
from neutron_lib import constants as n_const
import requests
import testtools
import webob
@ -469,10 +470,30 @@ class _TestMetadataProxyHandlerCacheMixin(object):
self.assertIsInstance(self._proxy_request_test_helper(500),
webob.exc.HTTPInternalServerError)
def test_proxy_request_502(self):
self.assertIsInstance(self._proxy_request_test_helper(502),
webob.exc.HTTPBadGateway)
def test_proxy_request_503(self):
self.assertIsInstance(self._proxy_request_test_helper(503),
webob.exc.HTTPServiceUnavailable)
def test_proxy_request_504(self):
self.assertIsInstance(self._proxy_request_test_helper(504),
webob.exc.HTTPGatewayTimeout)
def test_proxy_request_other_code(self):
with testtools.ExpectedException(Exception):
self._proxy_request_test_helper(302)
def test_proxy_request_conenction_error(self):
req = mock.Mock(path_info='/the_path', query_string='', headers={},
method='GET', body='')
with mock.patch('requests.request') as mock_request:
mock_request.side_effect = requests.ConnectionError()
retval = self.handler._proxy_request('the_id', 'tenant_id', req)
self.assertIsInstance(retval, webob.exc.HTTPServiceUnavailable)
class TestMetadataProxyHandlerNewCache(TestMetadataProxyHandlerBase,
_TestMetadataProxyHandlerCacheMixin):

21
neutron/tests/unit/agent/ovn/metadata/test_server.py
View File

@ -18,6 +18,7 @@ from unittest import mock
from oslo_config import cfg
from oslo_config import fixture as config_fixture
from oslo_utils import fileutils
import requests
import testtools
import webob
@ -232,10 +233,30 @@ class TestMetadataProxyHandler(base.BaseTestCase):
self.assertIsInstance(self._proxy_request_test_helper(500),
webob.exc.HTTPInternalServerError)
def test_proxy_request_502(self):
self.assertIsInstance(self._proxy_request_test_helper(502),
webob.exc.HTTPBadGateway)
def test_proxy_request_503(self):
self.assertIsInstance(self._proxy_request_test_helper(503),
webob.exc.HTTPServiceUnavailable)
def test_proxy_request_504(self):
self.assertIsInstance(self._proxy_request_test_helper(504),
webob.exc.HTTPGatewayTimeout)
def test_proxy_request_other_code(self):
with testtools.ExpectedException(Exception):
self._proxy_request_test_helper(302)
def test_proxy_request_conenction_error(self):
req = mock.Mock(path_info='/the_path', query_string='', headers={},
method='GET', body='')
with mock.patch('requests.request') as mock_request:
mock_request.side_effect = requests.ConnectionError()
retval = self.handler._proxy_request('the_id', 'tenant_id', req)
self.assertIsInstance(retval, webob.exc.HTTPServiceUnavailable)
class TestUnixDomainMetadataProxy(base.BaseTestCase):
def setUp(self):

6
releasenotes/notes/metadata-proxy-enhance-error-handling-3655404d44249097.yaml Normal file
View File

@ -0,0 +1,6 @@
---
other:
- |
Enhance error handling in the Neutron metadata service for cases when the
Nova metadata service is unavailable, ensuring correct HTTP status codes
are returned.