Optimize reconfiguration for mariadb

Change-Id: I278609f9832955849bc9381120a1b260f5a03f1b
Partially-implements: blueprint better-reconfigure

ansible/roles/mariadb/defaults/main.yml

@@ -1,6 +1,19 @@
 ---
 project_name: "mariadb"
 
+mariadb_services:
+  mariadb:
+    container_name: mariadb
+    group: mariadb
+    enabled: true
+    image: "{{ mariadb_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "mariadb:/var/lib/mysql"
+      - "kolla_logs:/var/log/kolla/"
+
+
 ####################
 # Database
 ####################

ansible/roles/mariadb/handlers/main.yml

@@ -0,0 +1,120 @@
+---
+- name: Starting first MariaDB container
+  vars:
+    service_name: "mariadb"
+    service: "{{ mariadb_services[service_name] }}"
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    environment:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+      BOOTSTRAP_ARGS: "--wsrep-new-cluster"
+    image: "{{ service.image }}"
+    labels:
+      BOOTSTRAP:
+    name: "{{ service.container_name }}"
+    restart_policy: "never"
+    volumes: "{{ service.volumes }}"
+  when:
+    - bootstrap_host is defined
+    - bootstrap_host == inventory_hostname
+  notify:
+    - wait first mariadb container
+    - restart slave mariadb
+    - restart master mariadb
+
+
+# TODO(jeffrey4l), remove the task check when the wait_for bug is fixed
+# https://github.com/ansible/ansible-modules-core/issues/2788
+- name: wait first mariadb container
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ mariadb_port }}"
+    connect_timeout: 1
+    timeout: 60
+    search_regex: "MariaDB"
+  register: check_mariadb_port
+  until: check_mariadb_port | success
+  retries: 10
+  delay: 6
+  when:
+    - bootstrap_host is defined
+    - bootstrap_host == inventory_hostname
+
+- name: restart slave mariadb
+  vars:
+    service_name: "mariadb"
+    service: "{{ mariadb_services[service_name] }}"
+    mariadb_container: "{{ check_mariadb_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+  when:
+    - inventory_hostname != master_host
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - mariadb_config_json.changed | bool
+      or mariadb_galera_conf.changed | bool
+      or mariadb_wsrep_notify.changed | bool
+      or mariadb_container.changed | bool
+      or bootstrap_host is defined
+  notify:
+    - wait for slave mariadb
+
+# TODO(jeffrey4l), remove the task check when the wait_for bug is fixed
+# https://github.com/ansible/ansible-modules-core/issues/2788
+- name: wait for slave mariadb
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ mariadb_port }}"
+    connect_timeout: 1
+    timeout: 60
+    search_regex: "MariaDB"
+  register: check_mariadb_port
+  until: check_mariadb_port | success
+  retries: 10
+  delay: 6
+  when:
+    - inventory_hostname != master_host
+
+- name: restart master mariadb
+  vars:
+    service_name: "mariadb"
+    service: "{{ mariadb_services[service_name] }}"
+    mariadb_container: "{{ check_mariadb_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+  when:
+    - inventory_hostname == master_host
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - mariadb_config_json.changed | bool
+      or mariadb_galera_conf.changed | bool
+      or mariadb_wsrep_notify.changed | bool
+      or mariadb_container.changed | bool
+      or bootstrap_host is defined
+  notify:
+    - Waiting for master mariadb
+
+# TODO(jeffrey4l), remove the task check when the wait_for bug is fixed
+# https://github.com/ansible/ansible-modules-core/issues/2788
+- name: Waiting for master mariadb
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ mariadb_port }}"
+    connect_timeout: 1
+    timeout: 60
+    search_regex: "MariaDB"
+  register: check_mariadb_port
+  until: check_mariadb_port | success
+  retries: 10
+  delay: 6
+  when:
+    - inventory_hostname == master_host

ansible/roles/mariadb/tasks/bootstrap.yml

@@ -1,9 +1,12 @@
 ---
+- set_fact:
+    master_host: "{{ groups['mariadb'][0] }}"
+
 - include: lookup_cluster.yml
 
 - include: bootstrap_cluster.yml
   when:
-    - delegate_host == 'None'
+    - not has_cluster | bool
     - inventory_hostname == groups['mariadb'][0]
 
 - include: recover_cluster.yml

ansible/roles/mariadb/tasks/bootstrap_cluster.yml

@@ -19,21 +19,8 @@
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla/"
       - "mariadb:/var/lib/mysql"
+  notify:
+    - Starting first MariaDB container
 
-- name: Starting first MariaDB container
+- set_fact:
-  kolla_docker:
+    bootstrap_host: "{{ inventory_hostname }}"
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    environment:
-      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
-      BOOTSTRAP_ARGS: "--wsrep-new-cluster"
-    image: "{{ mariadb_image_full }}"
-    labels:
-      BOOTSTRAP:
-    name: "mariadb"
-    restart_policy: "never"
-    volumes:
-      - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro"
-      - "/etc/localtime:/etc/localtime:ro"
-      - "kolla_logs:/var/log/kolla/"
-      - "mariadb:/var/lib/mysql"

ansible/roles/mariadb/tasks/config.yml

@@ -1,34 +1,73 @@
 ---
 - name: Ensuring config directories exist
   file:
-    path: "{{ node_config_directory }}/{{ item }}"
+    path: "{{ node_config_directory }}/{{ item.key }}"
     state: "directory"
     recurse: yes
-  with_items:
+  when:
-    - "mariadb"
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ mariadb_services }}"
 
 - name: Copying over config.json files for services
+  vars:
+    service_name: "mariadb"
+    service: "{{ mariadb_services[service_name] }}"
   template:
-    src: "{{ item }}.json.j2"
+    src: "{{ service_name }}.json.j2"
-    dest: "{{ node_config_directory }}/{{ item }}/config.json"
+    dest: "{{ node_config_directory }}/{{ service_name }}/config.json"
-  with_items:
+  register: mariadb_config_json
-    - "mariadb"
+  when:
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+  notify:
+    - restart slave mariadb
+    - restart master mariadb
 
 - name: Copying over galera.cnf
   vars:
-    service_name: "{{ item }}"
+    service_name: "mariadb"
+    service: "{{ mariadb_services[service_name] }}"
   merge_configs:
     sources:
       - "{{ role_path }}/templates/galera.cnf.j2"
       - "{{ node_custom_config }}/galera.cnf"
       - "{{ node_custom_config }}/mariadb/{{ inventory_hostname }}/galera.cnf"
-    dest: "{{ node_config_directory }}/{{ item }}/galera.cnf"
+    dest: "{{ node_config_directory }}/{{ service_name }}/galera.cnf"
-  with_items:
+  register: mariadb_galera_conf
-    - "mariadb"
+  when:
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+  notify:
+    - restart slave mariadb
+    - restart master mariadb
 
 - name: Copying over wsrep-notify.sh
   template:
     src: "{{ role_path }}/templates/wsrep-notify.sh.j2"
-    dest: "{{ node_config_directory }}/{{ item }}/wsrep-notify.sh"
+    dest: "{{ node_config_directory }}/{{ item.key }}/wsrep-notify.sh"
-  with_items:
+  register: mariadb_wsrep_notify
-    - "mariadb"
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ mariadb_services }}"
+  notify:
+    - restart slave mariadb
+    - restart master mariadb
+
+- name: Check mariadb containers
+  kolla_docker:
+    action: "compare_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ item.value.container_name }}"
+    image: "{{ item.value.image }}"
+    volumes: "{{ item.value.volumes }}"
+  register: check_mariadb_containers
+  when:
+    - action != "config"
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ mariadb_services }}"
+  notify:
+    - restart slave mariadb
+    - restart master mariadb

ansible/roles/mariadb/tasks/deploy.yml

@@ -3,20 +3,9 @@
 
 - include: bootstrap.yml
 
-- include: start.yml
+- name: Flush handlers
+  meta: flush_handlers
 
-# We use register as a test to see whether the database is active and ready to
-# communicate. This run on all hosts that have a database and attempts to talk
-# to the local database rather than the vip,
-- include: register.yml
-
-# This will restart the container we initially used to bootstrap the cluster to
-# make it match the other containers environment-wise. This also prevents a
-# change from showing up when rerunning the playbooks
-- include: start.yml
-
-# Since the last start.yml may have recreated some containers we must wait and
-# check the health again to ensure the hosts are active.
 - include: register.yml
 
 # Test haproxy user through VIP

ansible/roles/mariadb/tasks/lookup_cluster.yml

@@ -6,7 +6,7 @@
   run_once: True
 
 - name: Creating temp file on localhost
-  local_action: copy content=None dest=/tmp/kolla_mariadb_cluster mode=0644
+  local_action: copy content='' dest=/tmp/kolla_mariadb_cluster mode=0644
   changed_when: False
   check_mode: no
   run_once: True
@@ -26,7 +26,7 @@
 
 - name: Registering host from temp file
   set_fact:
-    delegate_host: "{{ lookup('file', '/tmp/kolla_mariadb_cluster') }}"
+    has_cluster: "{{ lookup('file', '/tmp/kolla_mariadb_cluster') | length > 0 }}"
 
 - name: Cleaning up temp file on localhost
   local_action: file path=/tmp/kolla_mariadb_cluster state=absent

ansible/roles/mariadb/tasks/pull.yml

@@ -3,5 +3,8 @@
   kolla_docker:
     action: "pull_image"
     common_options: "{{ docker_common_options }}"
-    image: "{{ mariadb_image_full }}"
+    image: "{{ item.value.image }}"
-  when: inventory_hostname in groups['mariadb']
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ mariadb_services }}"

ansible/roles/mariadb/tasks/reconfigure.yml

@@ -1,66 +1,2 @@
 ---
-- name: Ensuring the containers up
+- include: deploy.yml
-  kolla_docker:
-    name: "{{ item.name }}"
-    action: "get_container_state"
-  register: container_state
-  failed_when: container_state.Running == false
-  when: inventory_hostname in groups[item.group]
-  with_items:
-    - { name: mariadb, group: mariadb }
-
-- include: config.yml
-
-- name: Check the configs
-  command: docker exec {{ item.name }} /usr/local/bin/kolla_set_configs --check
-  changed_when: false
-  failed_when: false
-  register: check_results
-  when: inventory_hostname in groups[item.group]
-  with_items:
-    - { name: mariadb, group: mariadb }
-
-# NOTE(jeffrey4l): when config_strategy == 'COPY_ALWAYS'
-# and container env['KOLLA_CONFIG_STRATEGY'] == 'COPY_ONCE',
-# just remove the container and start again
-- name: Containers config strategy
-  kolla_docker:
-    name: "{{ item.name }}"
-    action: "get_container_env"
-  register: container_envs
-  when: inventory_hostname in groups[item.group]
-  with_items:
-    - { name: mariadb, group: mariadb }
-
-- name: Remove the containers
-  kolla_docker:
-    name: "{{ item[0]['name'] }}"
-    action: "remove_container"
-  register: remove_containers
-  when:
-    - config_strategy == "COPY_ONCE" or item[1]['KOLLA_CONFIG_STRATEGY'] == 'COPY_ONCE'
-    - item[2]['rc'] == 1
-    - inventory_hostname in groups[item[0]['group']]
-  with_together:
-    - [{ name: mariadb, group: mariadb }]
-    - "{{ container_envs.results }}"
-    - "{{ check_results.results }}"
-
-- include: start.yml
-  when: remove_containers.changed
-
-- name: Restart containers
-  kolla_docker:
-    name: "{{ item[0]['name'] }}"
-    action: "restart_container"
-  when:
-    - config_strategy == 'COPY_ALWAYS'
-    - item[1]['KOLLA_CONFIG_STRATEGY'] != 'COPY_ONCE'
-    - item[2]['rc'] == 1
-    - inventory_hostname in groups[item[0]['group']]
-  with_together:
-    - [{ name: mariadb, group: mariadb }]
-    - "{{ container_envs.results }}"
-    - "{{ check_results.results }}"
-
-- include: check.yml

ansible/roles/mariadb/tasks/recover_cluster.yml

@@ -1,7 +1,7 @@
 ---
 - fail:
     msg: "MariaDB cluster was not found. Is your inventory correct?"
-  when: delegate_host == 'None'
+  when: not has_cluster | bool
 
 - name: Checking if and mariadb containers are running
   kolla_docker:
@@ -57,6 +57,15 @@
   when:
     - mariadb_recover_inventory_name is not defined
 
+- set_fact:
+    bootstrap_host: "{{ mariadb_recover_inventory_name }}"
+    master_host: "{{ mariadb_recover_inventory_name }}"
+  changed_when: true
+  notify:
+    - Starting first MariaDB container
+    - restart slave mariadb
+    - restart master mariadb
+
 - name: Cleaning up temp file on mariadb hosts
   file: path=/tmp/kolla_mariadb_grastate.dat state=absent
   changed_when: false
@@ -67,28 +76,3 @@
   changed_when: false
   check_mode: no
   run_once: true
-
-- name: Starting first MariaDB container
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    environment:
-      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
-      BOOTSTRAP_ARGS: "--wsrep-new-cluster"
-    image: "{{ mariadb_image_full }}"
-    labels:
-      BOOTSTRAP:
-    name: "mariadb"
-    restart_policy: "never"
-    volumes:
-      - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro"
-      - "/etc/localtime:/etc/localtime:ro"
-      - "kolla_logs:/var/log/kolla/"
-      - "mariadb:/var/lib/mysql"
-  when:
-    - (mariadb_recover_inventory_name is not defined and inventory_hostname == groups['mariadb'][0]) or
-      (mariadb_recover_inventory_name is defined and inventory_hostname == mariadb_recover_inventory_name)
-
-- name: Reset bootstrap fact
-  set_fact:
-    delegate_host: "None"

ansible/roles/mariadb/tasks/start.yml

@@ -1,31 +0,0 @@
----
-- name: Starting mariadb container
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    image: "{{ mariadb_image_full }}"
-    name: "mariadb"
-    volumes:
-      - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro"
-      - "/etc/localtime:/etc/localtime:ro"
-      - "mariadb:/var/lib/mysql"
-      - "kolla_logs:/var/log/kolla/"
-  when: delegate_host != 'None' or
-        ( groups['mariadb'] | length ) == 1 or
-        ( delegate_host == 'None' and mariadb_recover_inventory_name is not defined and inventory_hostname != groups['mariadb'][0] ) or
-        ( delegate_host == 'None' and mariadb_recover_inventory_name is defined and inventory_hostname != mariadb_recover_inventory_name )
-
-
-# TODO(jeffrey4l), remove the task check when the wair_for bug is fixed
-# https://github.com/ansible/ansible-modules-core/issues/2788
-- name: Waiting for MariaDB service to be ready
-  wait_for:
-    host: "{{ api_interface_address }}"
-    port: "{{ mariadb_port }}"
-    connect_timeout: 1
-    timeout: 60
-    search_regex: "MariaDB"
-  register: check_mariadb_port
-  until: check_mariadb_port | success
-  retries: 10
-  delay: 6

ansible/roles/mariadb/tasks/upgrade.yml

@@ -1,8 +1,2 @@
 ---
-- include: config.yml
+- include: deploy.yml
-
-- include: lookup_cluster.yml
-
-- include: start.yml
-
-- include: check.yml