From 61b231a9995a0fe3230302ce913ffde707393d81 Mon Sep 17 00:00:00 2001 From: Doug Wiegley Date: Thu, 14 Feb 2019 11:12:57 -0700 Subject: [PATCH] Change process name of neutron-server to match worker role We spawn a lot of neutron-servers, on all but the smallest systems. It's often hard to tell which are busy/overloaded or spinning. Add an option to set the process names to their role. This has a small chance of breaking existing scripting, depending how they're parsing ps output. Sample output: $ ps xw | grep neutron-server 1126 pts/2 S+ 0:00 grep --color=auto neutron-server 25355 ? Ss 0:26 /usr/bin/python /usr/local/bin/neutron-server \ --config-file /etc/neutron/neutron.conf \ --config-file /etc/neutron/plugins/ml2/ml2_conf.ini 25368 ? S 0:00 neutron-server: api worker 25369 ? S 0:00 neutron-server: api worker 25370 ? S 0:00 neutron-server: api worker 25371 ? S 0:00 neutron-server: api worker 25372 ? S 0:02 neutron-server: rpc worker 25373 ? S 0:02 neutron-server: rpc worker 25374 ? S 0:02 neutron-server: services worker The "normal" looking ps output is the main parent. Partial-Bug: #1816485 Depends-On: https://review.openstack.org/637119 Change-Id: I0e664a5f8e792d85b8f5483fb8c6f1cd59a677cd --- neutron/conf/common.py | 7 ++++++ neutron/service.py | 12 +++++----- neutron/tests/unit/test_service.py | 3 ++- neutron/tests/unit/test_wsgi.py | 4 ++-- neutron/worker.py | 18 +++++++++++++-- neutron/wsgi.py | 23 +++++++++++-------- ...setproctitle_workers-bc27a8baa5ef2279.yaml | 20 ++++++++++++++++ 7 files changed, 66 insertions(+), 21 deletions(-) create mode 100644 releasenotes/notes/setproctitle_workers-bc27a8baa5ef2279.yaml diff --git a/neutron/conf/common.py b/neutron/conf/common.py index 31a719ebc82..456ffebddb0 100644 --- a/neutron/conf/common.py +++ b/neutron/conf/common.py @@ -110,6 +110,13 @@ core_opts = [ cfg.IntOpt('send_events_interval', default=2, help=_('Number of seconds between sending events to nova if ' 'there are any events to send.')), + cfg.StrOpt('setproctitle', default='on', + help=_("Set process name to match child worker role. " + "Available options are: 'off' - retains the previous " + "behavior; 'on' - renames processes to " + "'neutron-server: role (original string)'; " + "'brief' - renames the same as 'on', but without the " + "original string, such as 'neutron-server: role'.")), cfg.StrOpt('ipam_driver', default='internal', help=_("Neutron IPAM (IP address management) driver to use. " "By default, the reference implementation of the " diff --git a/neutron/service.py b/neutron/service.py index 23c9162d776..17c181f5162 100644 --- a/neutron/service.py +++ b/neutron/service.py @@ -24,7 +24,6 @@ from neutron_lib import context from neutron_lib.db import api as session from neutron_lib.plugins import directory from neutron_lib import rpc as n_rpc -from neutron_lib import worker as neutron_worker from oslo_concurrency import processutils from oslo_config import cfg from oslo_log import log as logging @@ -38,6 +37,7 @@ import psutil from neutron.common import config from neutron.common import profiler from neutron.conf import service +from neutron import worker as neutron_worker from neutron import wsgi @@ -94,7 +94,7 @@ def serve_wsgi(cls): return service -class RpcWorker(neutron_worker.BaseWorker): +class RpcWorker(neutron_worker.NeutronBaseWorker): """Wraps a worker to be handled by ProcessLauncher""" start_listeners_method = 'start_rpc_listeners' @@ -107,7 +107,7 @@ class RpcWorker(neutron_worker.BaseWorker): self._servers = [] def start(self): - super(RpcWorker, self).start() + super(RpcWorker, self).start(desc="rpc worker") for plugin in self._plugins: if hasattr(plugin, self.start_listeners_method): try: @@ -220,7 +220,7 @@ def _get_plugins_workers(): ] -class AllServicesNeutronWorker(neutron_worker.BaseWorker): +class AllServicesNeutronWorker(neutron_worker.NeutronBaseWorker): def __init__(self, services, worker_process_count=1): super(AllServicesNeutronWorker, self).__init__(worker_process_count) self._services = services @@ -230,7 +230,7 @@ class AllServicesNeutronWorker(neutron_worker.BaseWorker): def start(self): for srv in self._services: self._launcher.launch_service(srv) - super(AllServicesNeutronWorker, self).start() + super(AllServicesNeutronWorker, self).start(desc="services worker") def stop(self): self._launcher.stop() @@ -322,7 +322,7 @@ def _run_wsgi(app_name): def run_wsgi_app(app): server = wsgi.Server("Neutron") server.start(app, cfg.CONF.bind_port, cfg.CONF.bind_host, - workers=_get_api_workers()) + workers=_get_api_workers(), desc="api worker") LOG.info("Neutron service started, listening on %(host)s:%(port)s", {'host': cfg.CONF.bind_host, 'port': cfg.CONF.bind_port}) return server diff --git a/neutron/tests/unit/test_service.py b/neutron/tests/unit/test_service.py index 66a41f8ffa3..1e51723e93d 100644 --- a/neutron/tests/unit/test_service.py +++ b/neutron/tests/unit/test_service.py @@ -80,7 +80,8 @@ class TestRunWsgiApp(base.BaseTestCase): service.run_wsgi_app(mock.sentinel.app) start_call = mock_server.return_value.start.call_args expected_call = mock.call( - mock.ANY, mock.ANY, mock.ANY, workers=expected_passed_value) + mock.ANY, mock.ANY, mock.ANY, desc='api worker', + workers=expected_passed_value) self.assertEqual(expected_call, start_call) def test_api_workers_zero(self): diff --git a/neutron/tests/unit/test_wsgi.py b/neutron/tests/unit/test_wsgi.py index 662b0165762..5c8d8c179a4 100644 --- a/neutron/tests/unit/test_wsgi.py +++ b/neutron/tests/unit/test_wsgi.py @@ -74,7 +74,7 @@ class TestWorkerService(TestServiceBase): _service.pool.spawn.return_value = None _app = mock.Mock() - workerservice = wsgi.WorkerService(_service, _app) + workerservice = wsgi.WorkerService(_service, _app, "on") workerservice.start() self.assertFalse(apimock.called) @@ -82,7 +82,7 @@ class TestWorkerService(TestServiceBase): _service = mock.Mock() _app = mock.Mock() - worker_service = wsgi.WorkerService(_service, _app) + worker_service = wsgi.WorkerService(_service, _app, "on") self._test_reset(worker_service) diff --git a/neutron/worker.py b/neutron/worker.py index 6d808352b22..81e711092d7 100644 --- a/neutron/worker.py +++ b/neutron/worker.py @@ -11,10 +11,24 @@ # under the License. from neutron_lib import worker +from oslo_config import cfg from oslo_service import loopingcall -class PeriodicWorker(worker.BaseWorker): +class NeutronBaseWorker(worker.BaseWorker): + + def __init__(self, worker_process_count=1, set_proctitle=None): + set_proctitle = set_proctitle or cfg.CONF.setproctitle + super(NeutronBaseWorker, self).__init__( + worker_process_count=worker_process_count, + set_proctitle=set_proctitle + ) + + def start(self, name="neutron-server", desc=None): + super(NeutronBaseWorker, self).start(name=name, desc=desc) + + +class PeriodicWorker(NeutronBaseWorker): """A worker that runs a function at a fixed interval.""" def __init__(self, check_func, interval, initial_delay): @@ -26,7 +40,7 @@ class PeriodicWorker(worker.BaseWorker): self._initial_delay = initial_delay def start(self): - super(PeriodicWorker, self).start() + super(PeriodicWorker, self).start(desc="periodic worker") if self._loop is None: self._loop = loopingcall.FixedIntervalLoopingCall(self._check_func) self._loop.start(interval=self._interval, diff --git a/neutron/wsgi.py b/neutron/wsgi.py index 0107123fb8a..8552b352c67 100644 --- a/neutron/wsgi.py +++ b/neutron/wsgi.py @@ -25,7 +25,6 @@ import eventlet.wsgi from neutron_lib import context from neutron_lib.db import api as db_api from neutron_lib import exceptions as exception -from neutron_lib import worker as neutron_worker from oslo_config import cfg import oslo_i18n from oslo_log import log as logging @@ -43,6 +42,7 @@ import webob.exc from neutron._i18n import _ from neutron.common import config from neutron.conf import wsgi as wsgi_config +from neutron import worker as neutron_worker CONF = cfg.CONF wsgi_config.register_socket_opts() @@ -58,19 +58,20 @@ def encode_body(body): return encodeutils.to_utf8(body) -class WorkerService(neutron_worker.BaseWorker): +class WorkerService(neutron_worker.NeutronBaseWorker): """Wraps a worker to be handled by ProcessLauncher""" - def __init__(self, service, application, disable_ssl=False, + def __init__(self, service, application, set_proctitle, disable_ssl=False, worker_process_count=0): - super(WorkerService, self).__init__(worker_process_count) + super(WorkerService, self).__init__(worker_process_count, + set_proctitle) self._service = service self._application = application self._disable_ssl = disable_ssl self._server = None - def start(self): - super(WorkerService, self).start() + def start(self, desc=None): + super(WorkerService, self).start(desc=desc) # When api worker is stopped it kills the eventlet wsgi server which # internally closes the wsgi server socket object. This server socket # object becomes not usable which leads to "Bad file descriptor" @@ -162,7 +163,7 @@ class Server(object): return sock - def start(self, application, port, host='0.0.0.0', workers=0): + def start(self, application, port, host='0.0.0.0', workers=0, desc=None): """Run a WSGI server with the given application.""" self._host = host self._port = port @@ -174,14 +175,16 @@ class Server(object): self._launch(application, workers) - def _launch(self, application, workers=0): - service = WorkerService(self, application, self.disable_ssl, workers) + def _launch(self, application, workers=0, desc=None): + set_proctitle = "off" if desc is None else CONF.setproctitle + service = WorkerService(self, application, set_proctitle, + self.disable_ssl, workers) if workers < 1: # The API service should run in the current process. self._server = service # Dump the initial option values cfg.CONF.log_opt_values(LOG, logging.DEBUG) - service.start() + service.start(desc=desc) systemd.notify_once() else: # dispose the whole pool before os.fork, otherwise there will diff --git a/releasenotes/notes/setproctitle_workers-bc27a8baa5ef2279.yaml b/releasenotes/notes/setproctitle_workers-bc27a8baa5ef2279.yaml new file mode 100644 index 00000000000..1506693df83 --- /dev/null +++ b/releasenotes/notes/setproctitle_workers-bc27a8baa5ef2279.yaml @@ -0,0 +1,20 @@ +features: + - Neutron child processes now set their process titles + to match their roles ('api worker', 'rpc worker', + 'periodic worker', 'services worker', or any other defined + by workers from out-of-tree plugins.) This behavior can be + disabled by setting the ``setproctitle`` config option in the + ``[default]`` section in neutron.conf to ``off``. The original + process string is also appended to the end, to help with + scripting that is looking for the old strings. There is also an + option called ``brief``, which results in much shorter and easier + to read process names. The default setting for this + option is ``on``, for a combination of backwards compatibility + and identifying different processes easily. The recommended + setting is ``brief``, once the deployer has verified that none + of their tooling depends on the older strings. +upgrade: + - The change to the process title happens by default with the new + ``setproctitle`` config option. The old string is still part of + the new process title, but any scripts looking for exact string + matches of the old string may need to be modified.