From f3350d4e130b17b4035a8b67420d7015fc6cc99b Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 25 Mar 2020 11:00:31 +0000
Subject: [PATCH] Fix HAProxy prechecks during scale-out with limit
Deploy HAProxy on one or more servers. Add another server to the
inventory in the haproxy group, and run the following:
kolla-ansible prechecks --limit <new host>
The following task will fail:
TASK [haproxy : Checking if kolla_internal_vip_address and
kolla_external_vip_address are not pingable from any node]
This happens because ansible does not execute on hosts where
haproxy/keepalived is running, and therefore does not know that the VIP
should be active.
This change skips VIP prechecks when not all HAProxy hosts are in the
play.
Closes-Bug: #1868986
Change-Id: Ifbc73806b768f76f803ab01c115a9e5c2e2492ac
---
ansible/roles/haproxy/tasks/precheck.yml | 226 ++++++------------
...proxy-limit-precheck-c56b3ac2331867ee.yaml | 6 +
2 files changed, 83 insertions(+), 149 deletions(-)
create mode 100644 releasenotes/notes/fix-haproxy-limit-precheck-c56b3ac2331867ee.yaml
diff --git a/ansible/roles/haproxy/tasks/precheck.yml b/ansible/roles/haproxy/tasks/precheck.yml
index 13dccc31d5..10fcbf700e 100644
--- a/ansible/roles/haproxy/tasks/precheck.yml
+++ b/ansible/roles/haproxy/tasks/precheck.yml
@@ -7,103 +7,29 @@
- keepalived
register: container_facts
-- name: Clearing temp kolla_keepalived_running file
- file:
- path: /tmp/kolla_keepalived_running
- state: absent
- delegate_to: localhost
- changed_when: False
- check_mode: no
- run_once: true
+- name: Group hosts by whether they are running keepalived
+ group_by:
+ key: "keepalived_running_{{ container_facts['keepalived'] is defined }}"
when:
- enable_keepalived | bool
-
-- name: Create empty temp kolla_keepalived_running file
- copy:
- content: None
- dest: /tmp/kolla_keepalived_running
- mode: 0644
- changed_when: False
- delegate_to: localhost
- check_mode: no
- run_once: true
- when:
- - enable_keepalived | bool
-
-- name: Getting hosts who is running keepalived
- copy:
- content: "{{ ansible_hostname }}"
- dest: /tmp/kolla_keepalived_running
- mode: 0644
- delegate_to: localhost
- changed_when: False
- check_mode: no
- when:
- inventory_hostname in groups['haproxy']
- - container_facts['keepalived'] is defined
- - enable_keepalived | bool
-- name: Registering host running keepalived
- set_fact:
- host_running_keepalived: "{{ lookup('file', '/tmp/kolla_keepalived_running') }}"
+- name: Group hosts by whether they are running HAProxy
+ group_by:
+ key: "haproxy_running_{{ container_facts['haproxy'] is defined }}"
when:
- - enable_keepalived | bool
-
-- name: Clearing temp kolla_keepalived_running file
- file:
- path: /tmp/kolla_keepalived_running
- state: absent
- delegate_to: localhost
- changed_when: False
- check_mode: no
- run_once: true
- when:
- - enable_keepalived | bool
-
-- name: Clearing temp kolla_haproxy_running file
- file:
- path: /tmp/kolla_haproxy_running
- state: absent
- delegate_to: localhost
- changed_when: False
- check_mode: no
- run_once: true
-
-- name: Create empty temp kolla_haproxy_running file
- copy:
- content: None
- dest: /tmp/kolla_haproxy_running
- mode: 0644
- delegate_to: localhost
- changed_when: False
- check_mode: no
- run_once: true
-
-- name: Getting hosts who is running haproxy
- copy:
- content: "{{ ansible_hostname }}"
- dest: /tmp/kolla_haproxy_running
- mode: 0644
- delegate_to: localhost
- changed_when: False
- check_mode: no
- when:
- - inventory_hostname in groups['haproxy']
- - container_facts['haproxy'] is defined
- enable_haproxy | bool
+ - inventory_hostname in groups['haproxy']
-- name: Registering host running haproxy
+- name: Set facts about whether we can run HAProxy and keepalived VIP prechecks
+ vars:
+ # NOTE(mgoddard): We can only reliably run this precheck if all hosts in
+ # the haproxy group are included in the batch. This may not be the case if
+ # using --limit or --serial.
+ all_hosts_in_batch: "{{ groups['haproxy'] | difference(ansible_play_batch) | list | length == 0 }}"
set_fact:
- host_running_haproxy: "{{ lookup('file', '/tmp/kolla_haproxy_running') }}"
-
-- name: Clearing temp kolla_haproxy_running file
- file:
- path: /tmp/kolla_haproxy_running
- state: absent
- delegate_to: localhost
- changed_when: False
- check_mode: no
- run_once: true
+ keepalived_vip_prechecks: "{{ all_hosts_in_batch and groups['keepalived_running_True'] is not defined }}"
+ haproxy_vip_prechecks: "{{ all_hosts_in_batch and groups['haproxy_running_True'] is not defined }}"
- name: Checking if external haproxy certificate exists
run_once: true
@@ -163,7 +89,7 @@
command: "{{ 'ping' if kolla_external_vip_address|ipv4 else 'ping6' }}"
when:
- enable_keepalived | bool
- - "host_running_keepalived == 'None'"
+ - keepalived_vip_prechecks
- enable_haproxy | bool
- name: Checking free port for HAProxy stats
@@ -199,7 +125,7 @@
state: stopped
when:
- enable_haproxy | bool
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- inventory_hostname in groups['haproxy']
- api_interface_address != kolla_internal_vip_address
@@ -240,7 +166,7 @@
- enable_aodh | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('aodh_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Barbican API HAProxy
wait_for:
@@ -253,7 +179,7 @@
- enable_barbican | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('barbican_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Blazar API HAProxy
wait_for:
@@ -266,7 +192,7 @@
- enable_blazar | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('blazar_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Cinder API HAProxy
wait_for:
@@ -279,7 +205,7 @@
- enable_cinder | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('cinder_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Cloudkitty API HAProxy
wait_for:
@@ -292,7 +218,7 @@
- enable_cloudkitty | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('cloudkitty_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Congress API HAProxy
wait_for:
@@ -305,7 +231,8 @@
- enable_congress | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('congress_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
+
- name: Checking free port for Cyborg API HAProxy
wait_for:
@@ -318,6 +245,7 @@
- enable_cyborg | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('cyborg_api') == -1
+ - haproxy_vip_prechecks
- name: Checking free port for Designate API HAProxy
wait_for:
@@ -330,7 +258,7 @@
- enable_designate | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('designate_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Elasticsearch HAProxy
wait_for:
@@ -343,7 +271,7 @@
- enable_elasticsearch | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('elasticsearch') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Glance API HAProxy
wait_for:
@@ -356,7 +284,7 @@
- enable_glance | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('glance_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Gnocchi API HAProxy
wait_for:
@@ -369,7 +297,7 @@
- enable_gnocchi | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('gnocchi_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Freezer API HAProxy
wait_for:
@@ -382,7 +310,7 @@
- enable_freezer | bool
- haproxy_stat.find('freezer_api') == -1
- inventory_hostname in groups['haproxy']
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Grafana server HAProxy
wait_for:
@@ -395,7 +323,7 @@
- enable_grafana | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('grafana_server') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Heat API HAProxy
wait_for:
@@ -408,7 +336,7 @@
- enable_heat | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('heat_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Heat API CFN HAProxy
wait_for:
@@ -421,7 +349,7 @@
- enable_heat | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('heat_api_cfn') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Horizon HAProxy
wait_for:
@@ -434,7 +362,7 @@
- enable_horizon | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('horizon') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Ironic API HAProxy
wait_for:
@@ -447,7 +375,7 @@
- enable_ironic | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('ironic_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Ironic Inspector HAProxy
wait_for:
@@ -460,7 +388,7 @@
- enable_ironic | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('ironic_inspector') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Karbor Admin HAProxy
wait_for:
@@ -473,7 +401,7 @@
- enable_karbor | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('karbor_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Keystone Admin HAProxy
wait_for:
@@ -486,7 +414,7 @@
- enable_keystone | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('keystone_admin') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Keystone Internal HAProxy
wait_for:
@@ -499,7 +427,7 @@
- enable_keystone | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('keystone_internal') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Keystone Public HAProxy
wait_for:
@@ -513,7 +441,7 @@
- enable_keystone | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('keystone_external') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Kibana HAProxy
wait_for:
@@ -526,7 +454,7 @@
- enable_kibana | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('kibana') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Magnum API HAProxy
wait_for:
@@ -539,7 +467,7 @@
- enable_magnum | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('magnum_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Manila API HAProxy
wait_for:
@@ -552,7 +480,7 @@
- enable_manila | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('manila_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for MariaDB HAProxy
wait_for:
@@ -565,7 +493,7 @@
- enable_mariadb | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('mariadb') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Masakari API HAProxy
wait_for:
@@ -578,7 +506,7 @@
- enable_masakari | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('masakari_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Mistral API HAProxy
wait_for:
@@ -591,7 +519,7 @@
- enable_mistral | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('mistral_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Monasca API internal HAProxy
wait_for:
@@ -604,7 +532,7 @@
- enable_monasca | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('monasca_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Monasca API public HAProxy
wait_for:
@@ -618,7 +546,7 @@
- enable_monasca | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('monasca_api_external') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Monasca Log API internal HAProxy
wait_for:
@@ -631,7 +559,7 @@
- enable_monasca | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('monasca_log_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Monasca Log API public HAProxy
wait_for:
@@ -645,7 +573,7 @@
- enable_monasca | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('monasca_log_api_external') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Monasca Grafana API internal HAProxy
wait_for:
@@ -658,7 +586,7 @@
- enable_monasca | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('monasca_grafana_server') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Monasca Grafana API public HAProxy
wait_for:
@@ -672,7 +600,7 @@
- enable_monasca | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('monasca_grafana_server_external') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Mongodb HAProxy
wait_for:
@@ -685,7 +613,7 @@
- enable_mongodb | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('mongodb') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Murano API HAProxy
wait_for:
@@ -698,7 +626,7 @@
- enable_murano | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('murano_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Neutron Server HAProxy
wait_for:
@@ -711,7 +639,7 @@
- enable_neutron | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('neutron_server') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Nova API HAProxy
wait_for:
@@ -724,7 +652,7 @@
- enable_nova | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('nova_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Nova Metadata HAProxy
wait_for:
@@ -737,7 +665,7 @@
- enable_nova | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('nova_metadata') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Nova NoVNC HAProxy
wait_for:
@@ -751,7 +679,7 @@
- nova_console == 'novnc'
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('nova_novncproxy') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Nova Serial Proxy HAProxy
wait_for:
@@ -765,7 +693,7 @@
- haproxy_stat.find('nova_serialconsole_proxy') == -1
- enable_nova_serialconsole_proxy | bool
- inventory_hostname in groups['haproxy']
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Nova Spice HTML5 HAProxy
wait_for:
@@ -779,7 +707,7 @@
- nova_console == 'spice'
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('nova_spicehtml5proxy') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Nova Placement API HAProxy
wait_for:
@@ -792,7 +720,7 @@
- enable_nova | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('placement_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Octavia API HAProxy
wait_for:
@@ -805,7 +733,7 @@
- enable_octavia | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('octavia_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Panko API HAProxy
wait_for:
@@ -818,7 +746,7 @@
- enable_panko | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('panko_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Qinling API HAProxy
wait_for:
@@ -831,7 +759,7 @@
- enable_qinling | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('qinling_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for RabbitMQ Management HAProxy
wait_for:
@@ -844,7 +772,7 @@
- enable_rabbitmq | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('rabbitmq_management') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for outward RabbitMQ Management HAProxy
wait_for:
@@ -856,7 +784,7 @@
- enable_outward_rabbitmq | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('outward_rabbitmq_management') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Sahara API HAProxy
wait_for:
@@ -869,7 +797,7 @@
- enable_sahara | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('sahara_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Searchlight API HAProxy
wait_for:
@@ -882,7 +810,7 @@
- enable_searchlight | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('searchlight_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Senlin API HAProxy
wait_for:
@@ -895,7 +823,7 @@
- enable_senlin | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('senlin_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Solum Application Deployment HAProxy
wait_for:
@@ -908,7 +836,7 @@
- enable_solum | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('solum_application_deployment') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Solum Image Builder HAProxy
wait_for:
@@ -921,7 +849,7 @@
- enable_solum | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('solum_image_builder') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Swift Proxy Server HAProxy
wait_for:
@@ -934,7 +862,7 @@
- enable_swift | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('swift_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Tacker Server HAProxy
wait_for:
@@ -947,7 +875,7 @@
- enable_tacker | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('tacker_server') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Trove API HAProxy
wait_for:
@@ -960,7 +888,7 @@
- enable_trove | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('trove_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Watcher API HAProxy
wait_for:
@@ -973,7 +901,7 @@
- enable_watcher | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('watcher_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Zun API HAProxy
wait_for:
@@ -986,7 +914,7 @@
- enable_zun | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('zun_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
- name: Checking free port for Vitrage API HAProxy
wait_for:
@@ -998,4 +926,4 @@
- enable_vitrage | bool
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('vitrage_api') == -1
- - "host_running_haproxy == 'None'"
+ - haproxy_vip_prechecks
diff --git a/releasenotes/notes/fix-haproxy-limit-precheck-c56b3ac2331867ee.yaml b/releasenotes/notes/fix-haproxy-limit-precheck-c56b3ac2331867ee.yaml
new file mode 100644
index 0000000000..d6b93145fa
--- /dev/null
+++ b/releasenotes/notes/fix-haproxy-limit-precheck-c56b3ac2331867ee.yaml
@@ -0,0 +1,6 @@
+---
+fixes:
+ - |
+ Fixes an issue with HAProxy prechecks when scaling out using ``--limit`` or
+ ``--serial``. `LP#1868986
+ <https://bugs.launchpad.net/kolla-ansible/+bug/1868986>`__.