From 1ea029a91d2cf94161982963904354ed97c5ed45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= <radoslaw.piliszek@gmail.com>
Date: Tue, 28 Jan 2020 09:40:21 +0100
Subject: [PATCH] Followup on MariaDB handling fixes

This fixes issues reported by Mark:
- possible failure with 4-node cluster (however unlikely)
- failure to stop all nodes from progressing when conditions are
  not valid (due to: "any_errors_fatal: False")

Change-Id: Ib6995bf4c99202c9813859b3d9e2f420448f0445
---
 ansible/roles/mariadb/handlers/main.yml       |  3 +-
 .../roles/mariadb/tasks/lookup_cluster.yml    | 32 ++++++++++++-------
 2 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/ansible/roles/mariadb/handlers/main.yml b/ansible/roles/mariadb/handlers/main.yml
index 4aaee273b1..79e52d9084 100644
--- a/ansible/roles/mariadb/handlers/main.yml
+++ b/ansible/roles/mariadb/handlers/main.yml
@@ -68,13 +68,14 @@
   when:
     - groups.mariadb_port_alive_True is defined
     - inventory_hostname in groups.mariadb_port_alive_True
-    - groups.mariadb_port_alive_True.index(inventory_hostname) % 3 == item
+    - groups.mariadb_port_alive_True.index(inventory_hostname) % 4 == item
     - kolla_action != "config"
   listen: restart mariadb
   loop:
     - 0
     - 1
     - 2
+    - 3
 
 - name: Start MariaDB on new nodes
   include_tasks: 'restart_services.yml'
diff --git a/ansible/roles/mariadb/tasks/lookup_cluster.yml b/ansible/roles/mariadb/tasks/lookup_cluster.yml
index 35f0b5b3f8..4e55ec2b3c 100644
--- a/ansible/roles/mariadb/tasks/lookup_cluster.yml
+++ b/ansible/roles/mariadb/tasks/lookup_cluster.yml
@@ -30,6 +30,15 @@
       group_by:
         key: mariadb_port_alive_{{ check_mariadb_port_liveness is success }}
 
+    - name: Fail on existing but stopped cluster
+      fail:
+        msg: MariaDB cluster exists but is stopped. Please start it using kolla-ansible mariadb_recovery
+      when:
+        # NOTE(yoctozepto): we allow single-node cluster to start
+        - groups['mariadb'] | length > 1
+        - mariadb_cluster_exists
+        - groups.mariadb_port_alive_True is not defined
+
     - block:
         - name: Check MariaDB service WSREP sync status
           become: true
@@ -48,18 +57,19 @@
         - name: Extract MariaDB service WSREP sync status
           set_fact:
             mariadb_sync_status: "{{ check_mariadb_sync_status.stdout.split('\t')[1] }}"
-
-        - name: Divide hosts by their MariaDB service WSREP sync status
-          group_by:
-            key: mariadb_sync_status_{{ mariadb_sync_status }}
-
-        - name: Fail when MariaDB service is not synced
-          fail:
-            msg: MariaDB service is not synced. Please wait for WSREP sync before proceeding.
-          when:
-            - groups.mariadb_sync_status_Synced is not defined or
-              inventory_hostname not in groups.mariadb_sync_status_Synced
       when:
         - groups.mariadb_port_alive_True is defined
         - inventory_hostname in groups.mariadb_port_alive_True
+
+    - name: Divide hosts by their MariaDB service WSREP sync status
+      group_by:
+        key: mariadb_sync_status_{{ mariadb_sync_status | default('NA') }}
+
+    - name: Fail when MariaDB services are not synced across the whole cluster
+      fail:
+        msg: MariaDB cluster is not synced. Please wait for WSREP sync before proceeding.
+      when:
+        - groups.mariadb_port_alive_True is defined
+        - groups.mariadb_sync_status_Synced is not defined or
+          groups.mariadb_port_alive_True | sort != groups.mariadb_sync_status_Synced | sort
   when: not mariadb_recover | default(False)