HAProxy: Switch to L7 Healthchecks

Address occasional issues where TCP connections appear healthy,
yet the web servers within containers fail to respond,
resulting in requests being sent to unhealthy servers.

Implemented for services I currently use.
- Aodh (OPTIONS to /, expects 2XX or 3XX)
- Barbican (OPTIONS to /, expects 2XX or 3XX)
- Blazar (OPTIONS to /, expects 401)
- Cinder API (OPTIONS to /, expects 2XX or 3XX)
- CloudKitty (OPTIONS to /, expects 2XX or 3XX)
- Designate (OPTIONS to /, expects 2XX or 3XX)
- Glance (OPTIONS to /, expects 2XX or 3XX)
- Gnocchi (OPTIONS to /, expects 2XX or 3XX)
- Grafana (OPTIONS to /, expects 2XX or 3XX)
- Heat (OPTIONS to /, expects 2XX or 3XX)
- Horizon (OPTIONS to /, expects 2XX or 3XX)
- Ironic (OPTIONS to /, expects 2XX or 3XX)
- Keystone (OPTIONS to /, expects 2XX or 3XX)
- Magnum (OPTIONS to /, expects 2XX or 3XX)
- Manila (OPTIONS to /, expects 2XX or 3XX)
- Masakari (OPTIONS to /, expects 2XX or 3XX)
- Mistral (OPTIONS to /, expects 2XX or 3XX)
- Nova API (OPTIONS to /, expects 2XX or 3XX)
- Nova Metadata (OPTIONS to /, expects 2XX or 3XX)
- Neutron (OPTIONS to /, expects 2XX or 3XX)
- Opensearch (OPTIONS to /, expects 2XX or 3XX)
- Opensearch Dashboards (OPTIONS to /, expects 401)
- Placement (GET to /, expects 2XX or 3XX)
- Prometheus (OPTIONS to /, expects 2XX or 3XX)
- Prometheus AlertManager (OPTIONS to /, expects 2XX or 3XX)
- Prometheus Openstack Exporter (OPTIONS to /, expects 2XX or 3XX)
- Prometheus Server (OPTIONS to /, expects 2XX or 3XX)
- Skyline API (OPTIONS to /docs, expects 2XX or 3XX)
- Skyline Console (GET to /, expects 2XX or 3XX)
- Swift (OPTIONS to /info, expects 2XX or 3XX)
- Trove (OPTIONS to /, expects 2XX or 3XX)
- Venus (OPTIONS to /, expects 2XX or 3XX)
- Watcher (GET to /, expects 2XX or 3XX)
- Zun (OPTIONS to /, expects 2XX or 3XX)

Change-Id: I839f7f1051182fe797394e5436571d64d5c5b5a4

ansible/roles/aodh/defaults/main.yml

@@ -15,6 +15,8 @@ aodh_services:
         external: false
         port: "{{ aodh_api_port }}"
         listen_port: "{{ aodh_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
       aodh_api_external:
         enabled: "{{ enable_aodh }}"
         mode: "http"
@@ -22,6 +24,8 @@ aodh_services:
         external_fqdn: "{{ aodh_external_fqdn }}"
         port: "{{ aodh_api_public_port }}"
         listen_port: "{{ aodh_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
   aodh-evaluator:
     container_name: aodh_evaluator
     group: aodh-evaluator

ansible/roles/barbican/defaults/main.yml

@@ -16,6 +16,8 @@ barbican_services:
         port: "{{ barbican_api_port }}"
         listen_port: "{{ barbican_api_listen_port }}"
         tls_backend: "{{ barbican_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk"
       barbican_api_external:
         enabled: "{{ enable_barbican }}"
         mode: "http"
@@ -24,6 +26,8 @@ barbican_services:
         port: "{{ barbican_api_public_port }}"
         listen_port: "{{ barbican_api_listen_port }}"
         tls_backend: "{{ barbican_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk"
   barbican-keystone-listener:
     container_name: barbican_keystone_listener
     group: barbican-keystone-listener

ansible/roles/blazar/defaults/main.yml

@@ -15,6 +15,11 @@ blazar_services:
         external: false
         port: "{{ blazar_api_port }}"
         listen_port: "{{ blazar_api_listen_port }}"
+        backend_http_extra:
+          ## All of Blazar's API endpoints are private.
+          ## So we should accept 401 as valid healthy response.
+          - "option httpchk"
+          - "http-check expect status 401"
       blazar_api_external:
         enabled: "{{ enable_blazar }}"
         mode: "http"
@@ -22,6 +27,11 @@ blazar_services:
         external_fqdn: "{{ blazar_external_fqdn }}"
         port: "{{ blazar_api_public_port }}"
         listen_port: "{{ blazar_api_listen_port }}"
+        backend_http_extra:
+          ## All of Blazar's API endpoints are private.
+          ## So we should accept 401 as valid healthy response.
+          - "option httpchk"
+          - "http-check expect status 401"
   blazar-manager:
     container_name: blazar_manager
     group: blazar-manager

ansible/roles/cinder/defaults/main.yml

@@ -16,6 +16,8 @@ cinder_services:
         port: "{{ cinder_api_port }}"
         listen_port: "{{ cinder_api_listen_port }}"
         tls_backend: "{{ cinder_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk"
       cinder_api_external:
         enabled: "{{ enable_cinder }}"
         mode: "http"
@@ -24,6 +26,8 @@ cinder_services:
         port: "{{ cinder_api_public_port }}"
         listen_port: "{{ cinder_api_listen_port }}"
         tls_backend: "{{ cinder_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk"
   cinder-scheduler:
     container_name: cinder_scheduler
     group: cinder-scheduler

ansible/roles/cloudkitty/defaults/main.yml

@@ -15,6 +15,8 @@ cloudkitty_services:
         external: false
         port: "{{ cloudkitty_api_port }}"
         listen_port: "{{ cloudkitty_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
       cloudkitty_api_external:
         enabled: "{{ enable_cloudkitty }}"
         mode: "http"
@@ -22,6 +24,8 @@ cloudkitty_services:
         external_fqdn: "{{ cloudkitty_external_fqdn }}"
         port: "{{ cloudkitty_api_public_port }}"
         listen_port: "{{ cloudkitty_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
   cloudkitty-processor:
     container_name: "cloudkitty_processor"
     group: "cloudkitty-processor"

ansible/roles/designate/defaults/main.yml

@@ -15,6 +15,8 @@ designate_services:
         external: false
         port: "{{ designate_api_port }}"
         listen_port: "{{ designate_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
       designate_api_external:
         enabled: "{{ enable_designate }}"
         mode: "http"
@@ -22,6 +24,8 @@ designate_services:
         external_fqdn: "{{ designate_external_fqdn }}"
         port: "{{ designate_api_public_port }}"
         listen_port: "{{ designate_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
   designate-backend-bind9:
     container_name: designate_backend_bind9
     group: designate-backend-bind9

ansible/roles/glance/defaults/main.yml

@@ -21,6 +21,7 @@ glance_services:
           - "timeout client {{ haproxy_glance_api_client_timeout }}"
         backend_http_extra:
           - "timeout server {{ haproxy_glance_api_server_timeout }}"
+          - "option httpchk"
         custom_member_list: "{{ haproxy_members.split(';') }}"
       glance_api_external:
         enabled: "{{ enable_glance | bool and not glance_enable_tls_backend | bool }}"
@@ -32,6 +33,7 @@ glance_services:
           - "timeout client {{ haproxy_glance_api_client_timeout }}"
         backend_http_extra:
           - "timeout server {{ haproxy_glance_api_server_timeout }}"
+          - "option httpchk"
         custom_member_list: "{{ haproxy_members.split(';') }}"
   glance-tls-proxy:
     container_name: glance_tls_proxy
@@ -52,6 +54,7 @@ glance_services:
           - "timeout client {{ haproxy_glance_api_client_timeout }}"
         backend_http_extra:
           - "timeout server {{ haproxy_glance_api_server_timeout }}"
+          - "option httpchk"
         custom_member_list: "{{ haproxy_tls_members.split(';') }}"
         tls_backend: "yes"
       glance_tls_proxy_external:
@@ -64,6 +67,7 @@ glance_services:
           - "timeout client {{ haproxy_glance_api_client_timeout }}"
         backend_http_extra:
           - "timeout server {{ haproxy_glance_api_server_timeout }}"
+          - "option httpchk"
         custom_member_list: "{{ haproxy_tls_members.split(';') }}"
         tls_backend: "yes"
 

ansible/roles/gnocchi/defaults/main.yml

@@ -15,6 +15,8 @@ gnocchi_services:
         external: false
         port: "{{ gnocchi_api_port }}"
         listen_port: "{{ gnocchi_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
       gnocchi_api_external:
         enabled: "{{ enable_gnocchi }}"
         mode: "http"
@@ -22,6 +24,8 @@ gnocchi_services:
         external_fqdn: "{{ gnocchi_external_fqdn }}"
         port: "{{ gnocchi_api_public_port }}"
         listen_port: "{{ gnocchi_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
   gnocchi-metricd:
     container_name: gnocchi_metricd
     group: gnocchi-metricd

ansible/roles/grafana/defaults/main.yml

@@ -14,6 +14,8 @@ grafana_services:
         external: false
         port: "{{ grafana_server_port }}"
         listen_port: "{{ grafana_server_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
       grafana_server_external:
         enabled: "{{ enable_grafana_external | bool }}"
         mode: "http"
@@ -21,6 +23,8 @@ grafana_services:
         external_fqdn: "{{ grafana_external_fqdn }}"
         port: "{{ grafana_server_public_port }}"
         listen_port: "{{ grafana_server_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
 
 ####################
 # Database

ansible/roles/heat/defaults/main.yml

@@ -16,6 +16,8 @@ heat_services:
         port: "{{ heat_api_port }}"
         listen_port: "{{ heat_api_listen_port }}"
         tls_backend: "{{ heat_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk"
       heat_api_external:
         enabled: "{{ enable_heat }}"
         mode: "http"
@@ -24,6 +26,8 @@ heat_services:
         port: "{{ heat_api_public_port }}"
         listen_port: "{{ heat_api_listen_port }}"
         tls_backend: "{{ heat_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk"
   heat-api-cfn:
     container_name: heat_api_cfn
     group: heat-api-cfn
@@ -40,6 +44,8 @@ heat_services:
         port: "{{ heat_api_cfn_port }}"
         listen_port: "{{ heat_api_cfn_listen_port }}"
         tls_backend: "{{ heat_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk"
       heat_api_cfn_external:
         enabled: "{{ enable_heat }}"
         mode: "http"
@@ -48,6 +54,8 @@ heat_services:
         port: "{{ heat_api_cfn_public_port }}"
         listen_port: "{{ heat_api_cfn_listen_port }}"
         tls_backend: "{{ heat_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk"
   heat-engine:
     container_name: heat_engine
     group: heat-engine

ansible/roles/horizon/defaults/main.yml

@@ -37,6 +37,7 @@ horizon_services:
           - "use_backend acme_client_back if { path_reg ^/.well-known/acme-challenge/.+ }"
         backend_http_extra:
           - "balance {{ 'roundrobin' if enable_memcached | bool or horizon_backend_database | bool else 'source' }}"
+          - "option httpchk"
         tls_backend: "{{ horizon_enable_tls_backend }}"
       horizon_redirect:
         enabled: "{{ enable_horizon | bool and kolla_enable_tls_internal | bool }}"
@@ -57,6 +58,7 @@ horizon_services:
           - "use_backend acme_client_back if { path_reg ^/.well-known/acme-challenge/.+ }"
         backend_http_extra:
           - "balance {{ 'roundrobin' if enable_memcached | bool or horizon_backend_database | bool else 'source' }}"
+          - "option httpchk"
         tls_backend: "{{ horizon_enable_tls_backend }}"
       horizon_external_redirect:
         enabled: "{{ enable_horizon | bool and kolla_enable_tls_external | bool }}"

ansible/roles/ironic/defaults/main.yml

@@ -16,6 +16,8 @@ ironic_services:
         port: "{{ ironic_api_port }}"
         listen_port: "{{ ironic_api_listen_port }}"
         tls_backend: "{{ ironic_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk"
       ironic_api_external:
         enabled: "{{ enable_ironic }}"
         mode: "http"
@@ -24,6 +26,8 @@ ironic_services:
         port: "{{ ironic_api_public_port }}"
         listen_port: "{{ ironic_api_listen_port }}"
         tls_backend: "{{ ironic_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk"
   ironic-conductor:
     container_name: ironic_conductor
     group: ironic-conductor
@@ -49,6 +53,8 @@ ironic_services:
         external: false
         port: "{{ ironic_inspector_port }}"
         listen_port: "{{ ironic_inspector_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
       ironic_inspector_external:
         enabled: "{{ enable_ironic }}"
         mode: "http"
@@ -56,6 +62,8 @@ ironic_services:
         external_fqdn: "{{ ironic_inspector_external_fqdn }}"
         port: "{{ ironic_inspector_public_port }}"
         listen_port: "{{ ironic_inspector_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
   ironic-tftp:
     container_name: ironic_tftp
     group: ironic-tftp

ansible/roles/keystone/defaults/main.yml

@@ -18,6 +18,7 @@ keystone_services:
         listen_port: "{{ keystone_internal_listen_port }}"
         backend_http_extra:
           - "balance {{ 'source' if enable_keystone_federation | bool else 'roundrobin' }}"
+          - "option httpchk"
       keystone_external:
         enabled: "{{ enable_keystone }}"
         mode: "http"
@@ -28,6 +29,7 @@ keystone_services:
         listen_port: "{{ keystone_public_listen_port }}"
         backend_http_extra:
           - "balance {{ 'source' if enable_keystone_federation | bool else 'roundrobin' }}"
+          - "option httpchk"
   keystone-ssh:
     container_name: "keystone_ssh"
     group: "keystone"

ansible/roles/magnum/defaults/main.yml

@@ -17,6 +17,8 @@ magnum_services:
         external: false
         port: "{{ magnum_api_port }}"
         listen_port: "{{ magnum_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
       magnum_api_external:
         enabled: "{{ enable_magnum }}"
         mode: "http"
@@ -24,6 +26,8 @@ magnum_services:
         external_fqdn: "{{ magnum_external_fqdn }}"
         port: "{{ magnum_api_public_port }}"
         listen_port: "{{ magnum_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
   magnum-conductor:
     container_name: magnum_conductor
     group: magnum-conductor

ansible/roles/manila/defaults/main.yml

@@ -15,6 +15,8 @@ manila_services:
         external: false
         port: "{{ manila_api_port }}"
         listen_port: "{{ manila_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
       manila_api_external:
         enabled: "{{ enable_manila }}"
         mode: "http"
@@ -22,6 +24,8 @@ manila_services:
         external_fqdn: "{{ manila_external_fqdn }}"
         port: "{{ manila_api_public_port }}"
         listen_port: "{{ manila_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
   manila-scheduler:
     container_name: "manila_scheduler"
     group: "manila-scheduler"

ansible/roles/masakari/defaults/main.yml

@@ -14,6 +14,8 @@ masakari_services:
         external: false
         port: "{{ masakari_api_port }}"
         listen_port: "{{ masakari_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
       masakari_api_external:
         enabled: "{{ enable_masakari }}"
         mode: "http"
@@ -21,6 +23,8 @@ masakari_services:
         external_fqdn: "{{ masakari_external_fqdn }}"
         port: "{{ masakari_api_public_port }}"
         listen_port: "{{ masakari_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
   masakari-engine:
     container_name: masakari_engine
     group: masakari-engine

ansible/roles/mistral/defaults/main.yml

@@ -15,6 +15,8 @@ mistral_services:
         external: false
         port: "{{ mistral_api_port }}"
         listen_port: "{{ mistral_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
       mistral_api_external:
         enabled: "{{ enable_mistral }}"
         mode: "http"
@@ -22,6 +24,8 @@ mistral_services:
         external_fqdn: "{{ mistral_external_fqdn }}"
         port: "{{ mistral_api_public_port }}"
         listen_port: "{{ mistral_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
   mistral-engine:
     container_name: mistral_engine
     group: mistral-engine

ansible/roles/neutron/defaults/main.yml

@@ -16,6 +16,8 @@ neutron_services:
         external: false
         port: "{{ neutron_server_port }}"
         listen_port: "{{ neutron_server_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
       neutron_server_external:
         enabled: "{{ enable_neutron | bool and not neutron_enable_tls_backend | bool }}"
         mode: "http"
@@ -23,6 +25,8 @@ neutron_services:
         external_fqdn: "{{ neutron_external_fqdn }}"
         port: "{{ neutron_server_public_port }}"
         listen_port: "{{ neutron_server_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
   neutron-openvswitch-agent:
     container_name: "neutron_openvswitch_agent"
     image: "{{ neutron_openvswitch_agent_image_full }}"

ansible/roles/nova/defaults/main.yml

@@ -17,6 +17,8 @@ nova_services:
         port: "{{ nova_api_port }}"
         listen_port: "{{ nova_api_listen_port }}"
         tls_backend: "{{ nova_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk"
       nova_api_external:
         enabled: "{{ enable_nova }}"
         mode: "http"
@@ -25,6 +27,8 @@ nova_services:
         port: "{{ nova_api_public_port }}"
         listen_port: "{{ nova_api_listen_port }}"
         tls_backend: "{{ nova_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk"
   nova-metadata:
     container_name: "nova_metadata"
     group: "nova-metadata"
@@ -41,6 +45,8 @@ nova_services:
         port: "{{ nova_metadata_port }}"
         listen_port: "{{ nova_metadata_listen_port }}"
         tls_backend: "{{ nova_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk"
       nova_metadata_external:
         enabled: "{{ nova_enable_external_metadata }}"
         mode: "http"
@@ -49,6 +55,8 @@ nova_services:
         port: "{{ nova_metadata_port }}"
         listen_port: "{{ nova_metadata_listen_port }}"
         tls_backend: "{{ nova_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk"
   nova-scheduler:
     container_name: "nova_scheduler"
     group: "nova-scheduler"

ansible/roles/opensearch/defaults/main.yml

@@ -18,6 +18,8 @@ opensearch_services:
         port: "{{ opensearch_port }}"
         frontend_http_extra:
           - "option dontlog-normal"
+        backend_http_extra:
+          - "option httpchk"
   opensearch-dashboards:
     container_name: opensearch_dashboards
     group: opensearch-dashboards
@@ -36,6 +38,9 @@ opensearch_services:
         port: "{{ opensearch_dashboards_port }}"
         auth_user: "{{ opensearch_dashboards_user }}"
         auth_pass: "{{ opensearch_dashboards_password }}"
+        backend_http_extra:
+          - "option httpchk"
+          - "http-check expect status 401"
       opensearch_dashboards_external:
         enabled: "{{ enable_opensearch_dashboards_external | bool }}"
         mode: "http"
@@ -45,6 +50,9 @@ opensearch_services:
         listen_port: "{{ opensearch_dashboards_listen_port }}"
         auth_user: "{{ opensearch_dashboards_user }}"
         auth_pass: "{{ opensearch_dashboards_password }}"
+        backend_http_extra:
+          - "option httpchk"
+          - "http-check expect status 401"
 
 
 ####################

ansible/roles/placement/defaults/main.yml

@@ -16,6 +16,8 @@ placement_services:
         port: "{{ placement_api_port }}"
         listen_port: "{{ placement_api_listen_port }}"
         tls_backend: "{{ placement_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk GET /"
       placement_api_external:
         enabled: "{{ enable_placement }}"
         mode: "http"
@@ -24,6 +26,8 @@ placement_services:
         port: "{{ placement_api_public_port }}"
         listen_port: "{{ placement_api_listen_port }}"
         tls_backend: "{{ placement_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk GET /"
 
 ####################
 # Config Validate

ansible/roles/prometheus/defaults/main.yml

@@ -14,6 +14,8 @@ prometheus_services:
         external: false
         port: "{{ prometheus_port }}"
         active_passive: "{{ prometheus_active_passive | bool }}"
+        backend_http_extra:
+          - "option httpchk"
       prometheus_server_external:
         enabled: "{{ enable_prometheus_server_external | bool }}"
         mode: "http"
@@ -22,6 +24,8 @@ prometheus_services:
         port: "{{ prometheus_public_port }}"
         listen_port: "{{ prometheus_listen_port }}"
         active_passive: "{{ prometheus_active_passive | bool }}"
+        backend_http_extra:
+          - "option httpchk"
   prometheus-node-exporter:
     container_name: prometheus_node_exporter
     group: prometheus-node-exporter
@@ -67,6 +71,8 @@ prometheus_services:
         auth_user: "{{ prometheus_alertmanager_user }}"
         auth_pass: "{{ prometheus_alertmanager_password }}"
         active_passive: "{{ prometheus_alertmanager_active_passive | bool }}"
+        backend_http_extra:
+          - "option httpchk"
       prometheus_alertmanager_external:
         enabled: "{{ enable_prometheus_alertmanager_external | bool }}"
         mode: "http"
@@ -77,6 +83,8 @@ prometheus_services:
         auth_user: "{{ prometheus_alertmanager_user }}"
         auth_pass: "{{ prometheus_alertmanager_password }}"
         active_passive: "{{ prometheus_alertmanager_active_passive | bool }}"
+        backend_http_extra:
+          - "option httpchk"
   prometheus-openstack-exporter:
     container_name: "prometheus_openstack_exporter"
     group: "prometheus-openstack-exporter"
@@ -93,6 +101,7 @@ prometheus_services:
         external: false
         port: "{{ prometheus_openstack_exporter_port }}"
         backend_http_extra:
+          - "option httpchk"
           - "timeout server {{ prometheus_openstack_exporter_timeout }}"
       prometheus_openstack_exporter_external:
         enabled: "{{ enable_prometheus_openstack_exporter_external | bool }}"
@@ -100,6 +109,7 @@ prometheus_services:
         external: true
         port: "{{ prometheus_openstack_exporter_port }}"
         backend_http_extra:
+          - "option httpchk"
           - "timeout server {{ prometheus_openstack_exporter_timeout }}"
   prometheus-elasticsearch-exporter:
     container_name: prometheus_elasticsearch_exporter

ansible/roles/skyline/defaults/main.yml

@@ -16,6 +16,8 @@ skyline_services:
         port: "{{ skyline_apiserver_port }}"
         listen_port: "{{ skyline_apiserver_listen_port }}"
         tls_backend: "{{ skyline_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk GET /docs"
       skyline_apiserver_external:
         enabled: "{{ enable_skyline }}"
         mode: "http"
@@ -24,6 +26,8 @@ skyline_services:
         port: "{{ skyline_apiserver_port }}"
         listen_port: "{{ skyline_apiserver_listen_port }}"
         tls_backend: "{{ skyline_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk GET /docs"
   skyline-console:
     container_name: skyline_console
     group: skyline-console
@@ -40,6 +44,8 @@ skyline_services:
         port: "{{ skyline_console_port }}"
         listen_port: "{{ skyline_console_listen_port }}"
         tls_backend: "{{ skyline_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk GET /"
       skyline_console_external:
         enabled: "{{ enable_skyline }}"
         mode: "http"
@@ -48,6 +54,8 @@ skyline_services:
         port: "{{ skyline_console_port }}"
         listen_port: "{{ skyline_console_listen_port }}"
         tls_backend: "{{ skyline_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk GET /"
 
 ####################
 # Database

ansible/roles/swift/defaults/main.yml

@@ -9,12 +9,16 @@ swift_services:
         mode: "http"
         external: false
         port: "{{ swift_proxy_server_listen_port }}"
+        backend_http_extra:
+          - "option httpchk /info"
       swift_api_external:
         enabled: "{{ enable_swift }}"
         mode: "http"
         external: true
         external_fqdn: "{{ swift_external_fqdn }}"
         port: "{{ swift_proxy_server_listen_port }}"
+        backend_http_extra:
+          - "option httpchk /info"
 
 ####################
 # Docker

ansible/roles/trove/defaults/main.yml

@@ -16,6 +16,8 @@ trove_services:
         port: "{{ trove_api_port }}"
         listen_port: "{{ trove_api_listen_port }}"
         tls_backend: "{{ trove_enable_tls_backend }}"
+        backend_http_extra:
+          - "option httpchk"
       trove_api_external:
         enabled: "{{ enable_trove }}"
         mode: "http"
@@ -24,6 +26,8 @@ trove_services:
         tls_backend: "{{ trove_enable_tls_backend }}"
         external_fqdn: "{{ trove_external_fqdn }}"
         port: "{{ trove_api_public_port }}"
+        backend_http_extra:
+          - "option httpchk"
   trove-conductor:
     container_name: trove_conductor
     group: trove-conductor

ansible/roles/venus/defaults/main.yml

@@ -14,12 +14,16 @@ venus_services:
         mode: "http"
         external: false
         port: "{{ venus_api_port }}"
+        backend_http_extra:
+          - "option httpchk"
       venus_api_external:
         enabled: "{{ enable_venus }}"
         mode: "http"
         external: true
         external_fqdn: "{{ venus_external_fqdn }}"
         port: "{{ venus_api_public_port }}"
+        backend_http_extra:
+          - "option httpchk"
   venus-manager:
     container_name: venus_manager
     group: venus-manager

ansible/roles/watcher/defaults/main.yml

@@ -15,6 +15,8 @@ watcher_services:
         external: false
         port: "{{ watcher_api_port }}"
         listen_port: "{{ watcher_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk GET /"
       watcher_api_external:
         enabled: "{{ enable_watcher }}"
         mode: "http"
@@ -22,6 +24,8 @@ watcher_services:
         external_fqdn: "{{ watcher_external_fqdn }}"
         port: "{{ watcher_api_public_port }}"
         listen_port: "{{ watcher_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk GET /"
   watcher-applier:
     container_name: watcher_applier
     group: watcher-applier

ansible/roles/zun/defaults/main.yml

@@ -15,6 +15,8 @@ zun_services:
         external: false
         port: "{{ zun_api_port }}"
         listen_port: "{{ zun_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
       zun_api_external:
         enabled: "{{ enable_zun }}"
         mode: "http"
@@ -22,6 +24,8 @@ zun_services:
         external_fqdn: "{{ zun_external_fqdn }}"
         port: "{{ zun_api_public_port }}"
         listen_port: "{{ zun_api_listen_port }}"
+        backend_http_extra:
+          - "option httpchk"
   zun-wsproxy:
     container_name: zun_wsproxy
     group: zun-wsproxy

releasenotes/notes/haproxy_healthchecks_to_l7-b05e8c7b177d1544.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Implement Layer 7 Healthchecks for HA Proxy.
+    This should fix traffic being sent to unhealthy servers in some scenarios.