Move online data migrations to post-restart step

It's supposed that online migrations are executed once services are
upgraded and restarted after upgrade. Eventually, you can run
online migrations before the next upgrade according to the doc [1]

So we move that to a separate file that is executed after all services
are upgraded and handlers are flushed. Tasks are delegated to API hosts
and we clean up facts for them as well.

[1] https://docs.openstack.org/cinder/latest/admin/upgrades.html#database-upgrades

Change-Id: Ic3ecdddd7dcc2dd617c8606278590c8e59230fdf

tasks/cinder_db_post_setup.yml

@@ -0,0 +1,59 @@
+---
+# Copyright 2023, Cleura AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Perform data migrations if necessary and all cinder services are up
+  block:
+
+    - name: Perform online data migrations
+      command: "{{ cinder_bin }}/cinder-manage db online_data_migrations"
+      register: data_migrations
+      changed_when: false
+      retries: 5
+      delay: 3
+      until: "data_migrations.rc in [0, 2]"
+
+  rescue:
+
+    - name: Check if all cinder services are up
+      shell: "{{ cinder_bin }}/cinder-manage service list | /usr/bin/awk '{ if(NR>1) print $5 }' | /bin/egrep -vq ':-)'"
+      register: failed_cinder_services
+      ignore_errors: True
+
+    - name: Fail if any cinder service is down
+      fail:
+        msg: At least one cinder service is down. You need to fix this before re-running the playbook.
+      when: failed_cinder_services.rc != 1
+
+    - name: All cinder services are up, but migration failed nontheless.
+      fail:
+        msg: An unexpected error occured.
+      when: failed_cinder_services.rc == 1
+
+  become: yes
+  become_user: "{{ cinder_system_user_name }}"
+  when:
+    - "ansible_local['openstack_ansible']['cinder']['need_online_data_migrations'] | bool"
+
+- name: Disable the online migrations requirement
+  delegate_to: "{{ item }}"
+  ini_file:
+    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
+    section: cinder
+    option: need_online_data_migrations
+    value: False
+  with_items: "{{ groups[cinder_services['cinder-api']['group']] }}"
+  when:
+    - not data_migrations is skipped
+    - data_migrations  is succeeded

tasks/cinder_db_sync.yml

@@ -12,49 +12,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 - name: Perform a cinder DB sync
   command: "{{ cinder_bin }}/cinder-manage db sync"
   become: yes
   become_user: "{{ cinder_system_user_name }}"
   changed_when: false
-
-- name: Perform data migrations if necessary and all cinder services are up
-  block:
-
-    - name: Perform online data migrations
-      command: "{{ cinder_bin }}/cinder-manage db online_data_migrations"
-      register: data_migrations
-      changed_when: false
-
-  rescue:
-
-    - name: Check if all cinder services are up
-      shell: "{{ cinder_bin }}/cinder-manage service list | /usr/bin/awk '{ if(NR>1) print $5 }' | /bin/egrep -vq ':-)'"
-      register: failed_cinder_services
-      ignore_errors: True
-
-    - name: Fail if any cinder service is down
-      fail:
-        msg: At least one cinder service is down. You need to fix this before re-running the playbook.
-      when: failed_cinder_services.rc != 1
-
-    - name: All cinder services are up, but migration failed nontheless.
-      fail:
-        msg: An unexpected error occured.
-      when: failed_cinder_services.rc == 1
-
-  become: yes
-  become_user: "{{ cinder_system_user_name }}"
-  when:
-    - "(cinder_all_software_updated | default('no')) | bool"
-    - "ansible_local['openstack_ansible']['cinder']['need_online_data_migrations'] | bool"
-
-- name: Disable the online migrations requirement
-  ini_file:
-    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
-    section: cinder
-    option: need_online_data_migrations
-    value: False
-  when:
-    - not data_migrations is skipped
-    - data_migrations  is succeeded

tasks/main.yml

@@ -216,6 +216,12 @@
 - name: Flush handlers
   meta: flush_handlers
 
+- import_tasks: cinder_db_post_setup.yml
+  when:
+    - "_cinder_is_last_play_host"
+  tags:
+    - cinder-config
+
 - import_tasks: cinder_backends.yml
   when:
     - "groups[cinder_services['cinder-volume']['group']] | length > 0"