From 756618d250e8b61afd0ea6b6bc5c351d19d1a099 Mon Sep 17 00:00:00 2001 From: Dmitriy Rabotyagov Date: Tue, 14 Jan 2025 11:43:19 +0100 Subject: [PATCH] Optimize generation of required roles/collections Instead of running set_fact in cycle which consumes significant amount of output and time to execute, patch moves roles and collections generation process into a simplier Jinja statements which are easier to asses and quicker to execute. Change-Id: I943a608c56a68b32f5ee8d47b10596f35393cae5 --- .../get-ansible-collection-requirements.yml | 32 +++++-------------- scripts/get-ansible-role-requirements.yml | 26 ++++++--------- 2 files changed, 17 insertions(+), 41 deletions(-) diff --git a/scripts/get-ansible-collection-requirements.yml b/scripts/get-ansible-collection-requirements.yml index 5d1de660e6..2191929679 100644 --- a/scripts/get-ansible-collection-requirements.yml +++ b/scripts/get-ansible-collection-requirements.yml @@ -25,26 +25,7 @@ state: directory recurse: yes - - name: Read the list of user collections - set_fact: - user_collection_names: "{{ user_collections.collections | default([]) | map(attribute='name') | list }}" - - - name: Generate a list of required collections excluding user overridden collections - set_fact: - galaxy_collections_list: "{{ (galaxy_collections_list | default([])) + [item] }}" - when: - - item.name not in user_collection_names - with_items: "{{ required_collections.collections }}" - - - name: Append user collections to filtered required collections - set_fact: - galaxy_collections_list: "{{ (galaxy_collections_list | default([])) + [item] }}" - with_items: "{{ user_collections.collections }}" - when: - - user_collections.collections is defined - - "'source' in item" - - - name: Installing resulting collection list + - name: Installing required Ansible collections block: - name: Create temporary file for galaxy collection requirements tempfile: @@ -52,14 +33,17 @@ - name: Copy content into galaxy collection requirements temporary file vars: + _user_collection_names: "{{ user_collections.collections | default([]) | map(attribute='name') }}" + _default_collections: "{{ required_collections.collections | rejectattr('name', 'in', _user_collection_names) }}" + _user_collections: "{{ user_collections.collections | default([]) | selectattr('source', 'defined') }}" content_var: - collections: "{{ galaxy_collections_list }}" + collections: "{{ _default_collections + _user_collections }}" copy: content: "{{ content_var | to_nice_yaml }}" dest: "{{ collection_requirements_tmpfile.path }}" mode: "0644" - - name: Install collection requirements with ansible galaxy + - name: Install collection requirements with ansible galaxy # noqa: no-changed-when command: > /opt/ansible-runtime/bin/ansible-galaxy collection install --force -r "{{ collection_requirements_tmpfile.path }}" @@ -71,7 +55,7 @@ - name: Show collection install output debug: - msg: "{{ collection_install.stdout.split('\n') }}" + msg: "{{ collection_install.stdout_lines }}" always: - name: Clean up temporary file @@ -84,5 +68,5 @@ required_collections: "{{ lookup('file', collection_file) | from_yaml }}" collection_path_default: '/etc/ansible/' user_collection_file: 'user-collection-requirements.yml' - user_collections: "{{ lookup('file', user_collections_path, errors='ignore')|default([], true) | from_yaml }}" + user_collections: "{{ lookup('file', user_collections_path, errors='ignore') | default({}, true) | from_yaml }}" user_collections_path: "{{ lookup('env', 'OSA_CONFIG_DIR') | default('/etc/openstack_deploy', true) ~ '/' ~ user_collection_file }}" diff --git a/scripts/get-ansible-role-requirements.yml b/scripts/get-ansible-role-requirements.yml index 80909a8639..4f55b127ed 100644 --- a/scripts/get-ansible-role-requirements.yml +++ b/scripts/get-ansible-role-requirements.yml @@ -37,6 +37,14 @@ name: http.https://opendev.org/.userAgent value: "{{ 'git/' ~ _git_version.stdout.split(' ')[2] ~ ' (osa/' ~ lookup('env', 'CURRENT_OSA_VERSION') ~ '/deploy)' }}" + - name: Generate a list of roles to clone + vars: + _default_roles: "{{ required_roles | rejectattr('name', 'in', user_roles | map(attribute='name')) }}" + _user_roles_filtered: "{{ user_roles | rejectattr('src', 'undefined') }}" + _role_list: "{{ _default_roles + _user_roles_filtered }}" + set_fact: + clone_roles: "{{ _role_list | selectattr('scm', 'undefined') + _role_list | selectattr('scm', 'eq', 'git') }}" + - name: Remove target directory if required file: path: "{{ item.path | default(role_path_default) }}/{{ item.name | default(item.src | basename) }}" @@ -44,7 +52,7 @@ when: - ((item.path | default(role_path_default) ~ '/' ~ item.name | default(item.src | basename) ~ '/.git') is not directory) or (lookup('env', 'DROP_ROLE_DIRS') | bool is true) - with_items: "{{ required_roles }}" + with_items: "{{ clone_roles }}" - name: Ensure the default roles directory exists file: @@ -52,22 +60,6 @@ state: directory mode: "0755" - - name: Generate a list of user overridden roles - set_fact: - user_overridden_roles: "{{ user_roles | json_query('[*].name') }}" - - - name: Generate a list of roles excluding user overridden roles - set_fact: - clone_roles: "{{ (clone_roles | default([])) + [item] }}" - when: - - item.scm == "git" or item.scm is undefined - - item.name not in user_overridden_roles - with_items: "{{ required_roles }}" - - - name: Append user overridden roles - set_fact: - clone_roles: "{{ (clone_roles | default([])) + user_roles | rejectattr('src', 'undefined') }}" - - name: Clone git repos block: - name: Clone git repos (parallel)