Optimize generation of required roles/collections

Instead of running set_fact in cycle which consumes significant amount
of output and time to execute, patch moves roles and collections
generation process into a simplier Jinja statements
which are easier to asses and quicker to execute.

Change-Id: I943a608c56a68b32f5ee8d47b10596f35393cae5

scripts/get-ansible-collection-requirements.yml

@@ -25,26 +25,7 @@
         state: directory
         recurse: yes
 
-    - name: Read the list of user collections
+    - name: Installing required Ansible collections
-      set_fact:
-        user_collection_names: "{{ user_collections.collections | default([]) | map(attribute='name') | list }}"
-
-    - name: Generate a list of required collections excluding user overridden collections
-      set_fact:
-        galaxy_collections_list: "{{ (galaxy_collections_list | default([])) + [item] }}"
-      when:
-        - item.name not in user_collection_names
-      with_items: "{{ required_collections.collections }}"
-
-    - name: Append user collections to filtered required collections
-      set_fact:
-        galaxy_collections_list: "{{ (galaxy_collections_list | default([])) + [item] }}"
-      with_items: "{{ user_collections.collections }}"
-      when:
-        - user_collections.collections is defined
-        - "'source' in item"
-
-    - name: Installing resulting collection list
       block:
         - name: Create temporary file for galaxy collection requirements
           tempfile:
@@ -52,14 +33,17 @@
 
         - name: Copy content into galaxy collection requirements temporary file
           vars:
+            _user_collection_names: "{{ user_collections.collections | default([]) | map(attribute='name') }}"
+            _default_collections: "{{ required_collections.collections | rejectattr('name', 'in', _user_collection_names) }}"
+            _user_collections: "{{ user_collections.collections | default([]) | selectattr('source', 'defined') }}"
             content_var:
-              collections: "{{ galaxy_collections_list }}"
+              collections: "{{ _default_collections + _user_collections }}"
           copy:
             content: "{{ content_var | to_nice_yaml }}"
             dest: "{{ collection_requirements_tmpfile.path }}"
             mode: "0644"
 
-        - name: Install collection requirements with ansible galaxy
+        - name: Install collection requirements with ansible galaxy # noqa: no-changed-when
           command: >
             /opt/ansible-runtime/bin/ansible-galaxy collection install --force
             -r "{{ collection_requirements_tmpfile.path }}"
@@ -71,7 +55,7 @@
 
         - name: Show collection install output
           debug:
-            msg: "{{ collection_install.stdout.split('\n') }}"
+            msg: "{{ collection_install.stdout_lines }}"
 
       always:
         - name: Clean up temporary file
@@ -84,5 +68,5 @@
     required_collections: "{{ lookup('file', collection_file) | from_yaml }}"
     collection_path_default: '/etc/ansible/'
     user_collection_file: 'user-collection-requirements.yml'
-    user_collections: "{{ lookup('file', user_collections_path, errors='ignore')|default([], true) | from_yaml }}"
+    user_collections: "{{ lookup('file', user_collections_path, errors='ignore') | default({}, true) | from_yaml }}"
     user_collections_path: "{{ lookup('env', 'OSA_CONFIG_DIR') | default('/etc/openstack_deploy', true) ~ '/' ~ user_collection_file }}"

scripts/get-ansible-role-requirements.yml

@@ -37,6 +37,14 @@
         name: http.https://opendev.org/.userAgent
         value: "{{ 'git/' ~ _git_version.stdout.split(' ')[2] ~ ' (osa/' ~ lookup('env', 'CURRENT_OSA_VERSION') ~ '/deploy)' }}"
 
+    - name: Generate a list of roles to clone
+      vars:
+        _default_roles: "{{ required_roles | rejectattr('name', 'in', user_roles | map(attribute='name')) }}"
+        _user_roles_filtered: "{{ user_roles | rejectattr('src', 'undefined') }}"
+        _role_list: "{{ _default_roles + _user_roles_filtered }}"
+      set_fact:
+        clone_roles: "{{ _role_list | selectattr('scm', 'undefined') + _role_list | selectattr('scm', 'eq', 'git') }}"
+
     - name: Remove target directory if required
       file:
         path: "{{ item.path | default(role_path_default) }}/{{ item.name | default(item.src | basename) }}"
@@ -44,7 +52,7 @@
       when:
         - ((item.path | default(role_path_default) ~ '/' ~ item.name | default(item.src | basename) ~ '/.git') is not directory) or
           (lookup('env', 'DROP_ROLE_DIRS') | bool is true)
-      with_items: "{{ required_roles }}"
+      with_items: "{{ clone_roles }}"
 
     - name: Ensure the default roles directory exists
       file:
@@ -52,22 +60,6 @@
         state: directory
         mode: "0755"
 
-    - name: Generate a list of user overridden roles
-      set_fact:
-        user_overridden_roles: "{{ user_roles | json_query('[*].name') }}"
-
-    - name: Generate a list of roles excluding user overridden roles
-      set_fact:
-        clone_roles: "{{ (clone_roles | default([])) + [item] }}"
-      when:
-        - item.scm == "git" or item.scm is undefined
-        - item.name not in user_overridden_roles
-      with_items: "{{ required_roles }}"
-
-    - name: Append user overridden roles
-      set_fact:
-        clone_roles: "{{ (clone_roles | default([])) + user_roles | rejectattr('src', 'undefined') }}"
-
     - name: Clone git repos
       block:
         - name: Clone git repos (parallel)