From 1bcb139392a5989ef0cc733e1c39d7b6cc87ccbe Mon Sep 17 00:00:00 2001
From: Jeffrey Zhang <zhang.lei.fly@gmail.com>
Date: Sun, 9 Oct 2016 10:50:06 +0800
Subject: [PATCH] Choose node with largest seqno number for mariadb recovery

When all mariadb nodes are stopped gracefully, mariadb galera will
write it's last executed position into the grastate.dat file. Need find
the node with largest seqno number in that file and recovery from that
node.

Closes-Bug: #1627717
Change-Id: I6e97c190eec99c966bffde0698f783e519ba14bd
---
 .../roles/mariadb/tasks/recover_cluster.yml   | 54 +++++++++++++++++++
 ansible/roles/mariadb/tasks/start.yml         |  4 +-
 tools/kolla-ansible                           |  2 +-
 3 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/ansible/roles/mariadb/tasks/recover_cluster.yml b/ansible/roles/mariadb/tasks/recover_cluster.yml
index 2e0fcf5b46..f45b907cb3 100644
--- a/ansible/roles/mariadb/tasks/recover_cluster.yml
+++ b/ansible/roles/mariadb/tasks/recover_cluster.yml
@@ -14,6 +14,60 @@
   when: container_state.Running | bool
   any_errors_fatal: True
 
+- name: Cleaning up temp file on mariadb hosts
+  file: path=/tmp/kolla_mariadb_grastate.dat state=absent
+  changed_when: false
+  always_run: true
+
+- name: Cleaning up temp file on localhost
+  local_action: file path=/tmp/kolla_mariadb_recover_inventory_name state=absent
+  changed_when: false
+  always_run: true
+  run_once: true
+
+- block:
+  - name: Copying grastate.dat file from mariadb container
+    command: docker cp mariadb:/var/lib/mysql/grastate.dat /tmp/kolla_mariadb_grastate.dat
+    changed_when: false
+
+  - name: Print the content of grastate.dat file
+    command: cat /tmp/kolla_mariadb_grastate.dat
+    register: cat_grastate
+    changed_when: false
+
+  - name: Registering mariadb seqno variable
+    set_fact:
+      seqno: "{{ (cat_grastate.stdout|from_yaml).seqno }}"
+    changed_when: false
+
+  - name: Comparing seqno value on all mariadb hosts
+    shell: "if [[ {{ hostvars[inventory_hostname]['seqno'] }} -lt {{ hostvars[item]['seqno'] }} ]]; then echo {{ hostvars[item]['seqno'] }}; fi"
+    with_items: "{{ groups['mariadb'] }}"
+    changed_when: false
+    register: seqno_compare
+
+  - name: Writing hostname of host with the largest seqno to temp file
+    local_action: copy content={{ inventory_hostname }} dest=/tmp/kolla_mariadb_recover_inventory_name mode=0644
+    changed_when: false
+    when: seqno_compare.results | map(attribute='stdout') | join('') == ""
+
+  - name: Registering mariadb_recover_inventory_name from temp file
+    set_fact:
+      mariadb_recover_inventory_name: "{{ lookup('file', '/tmp/kolla_mariadb_recover_inventory_name') }}"
+  when:
+    - mariadb_recover_inventory_name is not defined
+
+- name: Cleaning up temp file on mariadb hosts
+  file: path=/tmp/kolla_mariadb_grastate.dat state=absent
+  changed_when: false
+  always_run: true
+
+- name: Cleaning up temp file on localhost
+  local_action: file path=/tmp/kolla_mariadb_recover_inventory_name state=absent
+  changed_when: false
+  always_run: true
+  run_once: true
+
 - name: Starting first MariaDB container
   kolla_docker:
     action: "start_container"
diff --git a/ansible/roles/mariadb/tasks/start.yml b/ansible/roles/mariadb/tasks/start.yml
index 3eff6ebeea..f71ad37a52 100644
--- a/ansible/roles/mariadb/tasks/start.yml
+++ b/ansible/roles/mariadb/tasks/start.yml
@@ -12,7 +12,9 @@
       - "kolla_logs:/var/log/kolla/"
   when: delegate_host != 'None' or
         ( groups['mariadb'] | length ) == 1 or
-        ( delegate_host == 'None' and inventory_hostname != groups['mariadb'][0] )
+        ( delegate_host == 'None' and mariadb_recover_inventory_name is not defined and inventory_hostname != groups['mariadb'][0] ) or
+        ( delegate_host == 'None' and mariadb_recover_inventory_name is defined and inventory_hostname != mariadb_recover_inventory_name )
+
 
 # TODO(jeffrey4l), remove the task check when the wair_for bug is fixed
 # https://github.com/ansible/ansible-modules-core/issues/2788
diff --git a/tools/kolla-ansible b/tools/kolla-ansible
index 21b3c82632..97a7c358cb 100755
--- a/tools/kolla-ansible
+++ b/tools/kolla-ansible
@@ -152,7 +152,7 @@ case "$1" in
         ;;
 (mariadb_recovery)
         ACTION="Attempting to restart mariadb cluster"
-        EXTRA_OPTS="$EXTRA_OPTS -e action=deploy"
+        EXTRA_OPTS="$EXTRA_OPTS -e action=deploy -e common_run=true"
         PLAYBOOK="${BASEDIR}/ansible/mariadb_recovery.yml"
         ;;
 (destroy)