Add variable for changing Apache HTTP timeout

In services which use the Apache HTTP server to service HTTP requests, there exists a TimeOut directive [1] which defaults to 60 seconds. APIs which come under heavy load, such as Cinder, can sometimes exceed this which results in a HTTP 504 Gateway timeout, or similar. However, the request can still be serviced without error. For example, if Nova calls the Cinder API to detach a volume, and this operation takes longer than the shortest of the two timeouts, Nova will emit a stack trace with a 504 Gateway timeout. At some time later, the request to detach the volume will succeed. The Nova and Cinder DBs then become out-of-sync with each other, and frequently DB surgery is required. Although strictly this category of bugs should be fixed in OpenStack services, it is not realistic to expect this to happen in the short term. Therefore, this change makes it easier to set the Apache HTTP timeout via a new variable. An example of a related bug is here: https://bugs.launchpad.net/nova/+bug/1888665 Whilst this timeout can currently be set by overriding the WSGI config for individual services, this change makes it much easier. Change-Id: Ie452516655cbd40d63bdad3635fd66693e40ce34 Closes-Bug: #1917648
2021-03-03 15:50:03 +00:00 · 2021-03-03 15:50:03 +00:00 · 647ff667e6
commit 647ff667e6
parent 1b1d06a9d7
19 changed files with 31 additions and 0 deletions
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@ -794,6 +794,7 @@ haproxy_backend_cacert_dir: "/etc/ssl/certs"
 # Backend options
 ##################
 kolla_httpd_keep_alive: "60"
+kolla_httpd_timeout: "60"

 ######################
 # Backend TLS options
--- a/ansible/roles/aodh/templates/wsgi-aodh.conf.j2
+++ b/ansible/roles/aodh/templates/wsgi-aodh.conf.j2
@ -5,6 +5,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ aodh_api_l
 ServerSignature Off
 ServerTokens Prod
 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 ErrorLog "{{ aodh_log_dir }}/apache-error.log"
--- a/ansible/roles/cinder/templates/cinder-wsgi.conf.j2
+++ b/ansible/roles/cinder/templates/cinder-wsgi.conf.j2
@ -10,6 +10,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ cinder_api
 ServerSignature Off
 ServerTokens Prod
 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 {% if cinder_logging_debug | bool %}
--- a/ansible/roles/cloudkitty/templates/wsgi-cloudkitty.conf.j2
+++ b/ansible/roles/cloudkitty/templates/wsgi-cloudkitty.conf.j2
@ -5,6 +5,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ cloudkitty
 ServerSignature Off
 ServerTokens Prod
 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 ErrorLog "{{ cloudkitty_log_dir }}/apache-error.log"
--- a/ansible/roles/freezer/templates/wsgi-freezer-api.conf.j2
+++ b/ansible/roles/freezer/templates/wsgi-freezer-api.conf.j2
@ -5,6 +5,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ freezer_ap
 ServerSignature Off
 ServerTokens Prod
 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 ErrorLog "{{ freezer_log_dir }}/apache-error.log"
--- a/ansible/roles/gnocchi/templates/wsgi-gnocchi.conf.j2
+++ b/ansible/roles/gnocchi/templates/wsgi-gnocchi.conf.j2
@ -5,6 +5,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ gnocchi_ap
 ServerSignature Off
 ServerTokens Prod
 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 ErrorLog "{{ gnocchi_log_dir }}/apache-error.log"
--- a/ansible/roles/heat/templates/wsgi-heat-api-cfn.conf.j2
+++ b/ansible/roles/heat/templates/wsgi-heat-api-cfn.conf.j2
@ -12,6 +12,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ heat_api_c
 ServerSignature Off
 ServerTokens Prod
 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 <Directory "{{ binary_path }}">
--- a/ansible/roles/heat/templates/wsgi-heat-api.conf.j2
+++ b/ansible/roles/heat/templates/wsgi-heat-api.conf.j2
@ -12,6 +12,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ heat_api_l
 ServerSignature Off
 ServerTokens Prod
 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 <Directory "{{ binary_path }}">
--- a/ansible/roles/ironic/templates/ironic-api-wsgi.conf.j2
+++ b/ansible/roles/ironic/templates/ironic-api-wsgi.conf.j2
@ -12,6 +12,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ ironic_api
 ServerSignature Off
 ServerTokens Prod
 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 <Directory "{{ wsgi_directory }}">
--- a/ansible/roles/keystone/templates/wsgi-keystone.conf.j2
+++ b/ansible/roles/keystone/templates/wsgi-keystone.conf.j2
@ -13,6 +13,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ keystone_a
 ServerSignature Off
 ServerTokens Prod
 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 ErrorLog "{{ keystone_log_dir }}/apache-error.log"
--- a/ansible/roles/masakari/templates/wsgi-masakari.conf.j2
+++ b/ansible/roles/masakari/templates/wsgi-masakari.conf.j2
@ -6,6 +6,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ masakari_a
 ServerSignature Off
 ServerTokens Prod
 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 ErrorLog "{{ masakari_log_dir }}/apache-error.log"
--- a/ansible/roles/monasca/templates/monasca-api/wsgi-api.conf.j2
+++ b/ansible/roles/monasca/templates/monasca-api/wsgi-api.conf.j2
@ -4,6 +4,7 @@
 Listen {{ api_interface_address | put_address_in_context('url') }}:{{ monasca_api_port }}

 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 ErrorLog "{{ monasca_log_dir }}/apache-api-error.log"
--- a/ansible/roles/nova/templates/nova-api-wsgi.conf.j2
+++ b/ansible/roles/nova/templates/nova-api-wsgi.conf.j2
@ -13,6 +13,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ nova_metad
 ServerSignature Off
 ServerTokens Prod
 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 <Directory "{{ wsgi_directory }}">
--- a/ansible/roles/panko/templates/wsgi-panko.conf.j2
+++ b/ansible/roles/panko/templates/wsgi-panko.conf.j2
@ -5,6 +5,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ panko_api_
 ServerSignature Off
 ServerTokens Prod
 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 ErrorLog "{{ panko_log_dir }}/apache-error.log"
--- a/ansible/roles/placement/templates/placement-api-wsgi.conf.j2
+++ b/ansible/roles/placement/templates/placement-api-wsgi.conf.j2
@ -12,6 +12,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ placement_
 ServerSignature Off
 ServerTokens Prod
 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 {% if placement_logging_debug | bool %}
--- a/ansible/roles/vitrage/templates/wsgi-vitrage.conf.j2
+++ b/ansible/roles/vitrage/templates/wsgi-vitrage.conf.j2
@ -4,6 +4,7 @@ Listen {{ 'api' | kolla_address | put_address_in_context('url') }}:{{ vitrage_ap
 ServerSignature Off
 ServerTokens Prod
 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 ErrorLog "{{ vitrage_log_dir }}/apache-error.log"
--- a/ansible/roles/zun/templates/wsgi-zun.conf.j2
+++ b/ansible/roles/zun/templates/wsgi-zun.conf.j2
@ -4,6 +4,7 @@ Listen {{ api_interface_address | put_address_in_context('url') }}:{{ zun_api_po
 ServerSignature Off
 ServerTokens Prod
 TraceEnable off
+TimeOut {{ kolla_httpd_timeout }}
 KeepAliveTimeout {{ kolla_httpd_keep_alive }}

 ErrorLog "{{ zun_log_dir }}/apache-error.log"
--- a/etc/kolla/globals.yml
+++ b/etc/kolla/globals.yml
@ -214,6 +214,7 @@
 # Backend options
 ##################
 #kolla_httpd_keep_alive: "60"
+#kolla_httpd_timeout: "60"

 #####################
 # Backend TLS options
--- a/releasenotes/notes/add-httpd-timeout-717050ffdc4ea9e8.yaml
+++ b/releasenotes/notes/add-httpd-timeout-717050ffdc4ea9e8.yaml
@ -0,0 +1,13 @@
+---
+features:
+  - |
+    Adds configuration parameter ``kolla_httpd_timeout`` to configure the
+    ``TimeOut`` directive for services that use Apache HTTP server to
+    handle HTTP requests. The default value is 60 seconds which matches
+    the original default, but you may wish to increase this.
+fixes:
+  - |
+    Fixes an issue where it was not easily possible to set the Apache
+    HTTP timeout directive, where the default of 60s would cause
+    problems in slow running services. See
+    `LP#1917648 <https://launchpad.net/bugs/1917648>`__.