From bc517cfcc5eb8f193dfd2770227df60b009194d7 Mon Sep 17 00:00:00 2001 From: Dmitriy Rabotyagov Date: Mon, 13 Jan 2025 19:29:45 +0100 Subject: [PATCH] Move zuul preparation for role/collection bootstrap At the moment bootstrap-ansible.sh playbooks are bloated with irrelevant for end-user and applicable only in CI tasks that ensure respect for depends-on in Zuul. Here it's proposed to move all CI-related activities from user facing scripts to Zull pre-tasks separately, which should remove confusing tasks from the bootstrap process and slightly speed-up its execution. Change-Id: I2f94bbf00719b23714bcf855127a0d22170f9064 --- scripts/gate-check-commit.sh | 7 +- .../get-ansible-collection-requirements.yml | 82 +++++------ scripts/get-ansible-role-requirements.yml | 65 +-------- zuul.d/jobs.yaml | 1 + zuul.d/playbooks/pre-osa-aio.yml | 2 + zuul.d/playbooks/pre-osa-requirements.yml | 132 ++++++++++++++++++ 6 files changed, 179 insertions(+), 110 deletions(-) create mode 100644 zuul.d/playbooks/pre-osa-requirements.yml diff --git a/scripts/gate-check-commit.sh b/scripts/gate-check-commit.sh index 48db00cca8..b821c9600d 100755 --- a/scripts/gate-check-commit.sh +++ b/scripts/gate-check-commit.sh @@ -256,7 +256,6 @@ if [[ "${ACTION}" =~ "upgrade" ]]; then # We need this as in stein we were deploying custom # /etc/openstack_deploy/env.d/aio_metal.yml for metal installs export SKIP_CUSTOM_ENVD_CHECK=true - export DROP_ROLE_DIRS=true # Export ZUUL_SRC_PATH only when integrated repo folder exists. Based on that # we make an assumption about if we're in CI or not @@ -264,6 +263,12 @@ if [[ "${ACTION}" =~ "upgrade" ]]; then export ZUUL_SRC_PATH="/home/zuul/src" # Doing symlinking here, as bootstrap role won't be called ln -s $ZUUL_SRC_PATH /openstack/src + # Run a zuul playbook to place use zuul-prepared roles/requirements to + # respect defined Depends-On on N. action=deploy is set as otherwise tasks will be skipped. + export DROP_ROLE_DIRS=true + ansible-playbook /home/zuul/src/opendev.org/openstack/openstack-ansible/zuul.d/playbooks/pre-osa-requirements.yml \ + -e pre_osa_host=localhost -e load_zuul_vars=False -e action=deploy + unset DROP_ROLE_DIRS fi # Update AIO config files for certain scenarios # for item in "${SCENARIOS_WITH_CONFIG_UPDATE[@]}"; do diff --git a/scripts/get-ansible-collection-requirements.yml b/scripts/get-ansible-collection-requirements.yml index 9a08d95804..5d1de660e6 100644 --- a/scripts/get-ansible-collection-requirements.yml +++ b/scripts/get-ansible-collection-requirements.yml @@ -29,73 +29,55 @@ set_fact: user_collection_names: "{{ user_collections.collections | default([]) | map(attribute='name') | list }}" - - name: Get Zuul cloned repos path in CI - set_fact: - zuul_src_path: "{{ lookup('env', 'ZUUL_SRC_PATH') }}" - - name: Generate a list of required collections excluding user overridden collections set_fact: - galaxy_collections_list : "{{ (galaxy_collections_list | default([])) + [ item ] }}" + galaxy_collections_list: "{{ (galaxy_collections_list | default([])) + [item] }}" when: - item.name not in user_collection_names with_items: "{{ required_collections.collections }}" - name: Append user collections to filtered required collections set_fact: - galaxy_collections_list: "{{ (galaxy_collections_list | default([])) + [ item ] }}" + galaxy_collections_list: "{{ (galaxy_collections_list | default([])) + [item] }}" with_items: "{{ user_collections.collections }}" when: - user_collections.collections is defined - "'source' in item" - - name: Check the Zuul src dir for cloned collections - stat: - path: "{{ zuul_src_path }}/{{ item.source.split('/')[2:] | join('/') | split('#') | first }}" - get_attributes: no - get_checksum: no - get_mime: no - register: zuul_collections - with_items: "{{ galaxy_collections_list }}" + - name: Installing resulting collection list + block: + - name: Create temporary file for galaxy collection requirements + tempfile: + register: collection_requirements_tmpfile - - name: Override paths for zuul hosted collections in CI - vars: - zuul_item: - name: "{{ item.item.name }}" - source: "{{ zuul_src_path }}/{{ item.item.source.split('/')[2:] | join('/') | replace('#', '') }}" - type: "dir" - set_fact: - galaxy_collections_list_ci: "{{ galaxy_collections_list_ci | default([]) + [(zuul_src_path and item.stat.exists) | ternary(zuul_item, item.item)] }}" - with_items: "{{ zuul_collections.results }}" + - name: Copy content into galaxy collection requirements temporary file + vars: + content_var: + collections: "{{ galaxy_collections_list }}" + copy: + content: "{{ content_var | to_nice_yaml }}" + dest: "{{ collection_requirements_tmpfile.path }}" + mode: "0644" - - name: Create temporary file for galaxy collection requirements - tempfile: - register: collection_requirements_tmpfile + - name: Install collection requirements with ansible galaxy + command: > + /opt/ansible-runtime/bin/ansible-galaxy collection install --force + -r "{{ collection_requirements_tmpfile.path }}" + -p "{{ collection_path_default }}" + register: collection_install + until: collection_install is success + retries: 5 + delay: 2 - - name: Copy content into galaxy collection requirements temporary file - vars: - content_var: - collections: "{{ galaxy_collections_list_ci }}" - copy: - content: "{{ content_var | to_nice_yaml }}" - dest: "{{ collection_requirements_tmpfile.path }}" + - name: Show collection install output + debug: + msg: "{{ collection_install.stdout.split('\n') }}" - - name: Install collection requirements with ansible galaxy - command: > - /opt/ansible-runtime/bin/ansible-galaxy collection install --force - -r "{{ collection_requirements_tmpfile.path }}" - -p "{{ collection_path_default }}" - register: collection_install - until: collection_install is success - retries: 5 - delay: 2 - - - name: Show collection install output - debug: msg="{{ collection_install.stdout.split('\n') }}" - - - name: Clean up temporary file - file: - path: "{{ collection_requirements_tmpfile.path }}" - state: absent + always: + - name: Clean up temporary file + file: + path: "{{ collection_requirements_tmpfile.path }}" + state: absent vars: collection_file: "{{ playbook_dir }}/../ansible-collection-requirements.yml" diff --git a/scripts/get-ansible-role-requirements.yml b/scripts/get-ansible-role-requirements.yml index 31a89dc075..80909a8639 100644 --- a/scripts/get-ansible-role-requirements.yml +++ b/scripts/get-ansible-role-requirements.yml @@ -50,56 +50,7 @@ file: path: "{{ role_path_default }}" state: directory - - - name: Use Zuul provided sources in Zuul environment - block: - - name: Set Zuul sources path - set_fact: - zuul_src_path: "{{ lookup('env', 'ZUUL_SRC_PATH') }}" - - name: Check the Zuul src dir for cloned roles - stat: - path: "{{ zuul_src_path }}/{{ item.src.split('/')[-3:] | join('/') }}" - get_attributes: no - get_checksum: no - get_mime: no - register: zuul_roles - when: - - item.scm == "git" or item.scm is undefined - with_items: "{{ required_roles }}" - - name: Link the Zuul provided roles - file: - src: "{{ zuul_src_path }}/{{ item.item.src.split('/')[-3:] | join('/') }}" - dest: "{{ item.item.path | default(role_path_default) }}/{{ item.item.name | default(item.item.src | basename) }}" - state: link - owner: root - group: root - with_items: "{{ zuul_roles.results - | selectattr('stat.exists') - | list }}" - # NOTE(mnaser): We need to make sure that all the roles - # are checked out by Zuul so we hard fail - # if any roles are not. - - name: Fail if any roles were not cloned - fail: - msg: | - The following roles were not cloned automatically by Zuul, - make sure that they're included in required-projects {{ uncloned_roles|join(',') }} - when: uncloned_roles | length > 0 - vars: - uncloned_roles: "{{ zuul_roles.results | rejectattr('stat.exists') - | map(attribute='item') - | map(attribute='src') - | select('match', 'opendev.org') - | list }}" - - - name: Ensure overrides directory exists - file: - path: "{{ config_dir }}" - state: directory - - when: - - "lookup('env', 'ZUUL_SRC_PATH') != ''" - - "lookup('env', 'UPGRADE_TARGET_BRANCH') == ''" + mode: "0755" - name: Generate a list of user overridden roles set_fact: @@ -107,15 +58,11 @@ - name: Generate a list of roles excluding user overridden roles set_fact: - clone_roles: "{{ (clone_roles | default([])) + [ item ] }}" + clone_roles: "{{ (clone_roles | default([])) + [item] }}" when: - item.scm == "git" or item.scm is undefined - item.name not in user_overridden_roles - with_items: "{{ (zuul_roles.results | default([]) | - selectattr('stat', 'defined') | - rejectattr('stat.exists') | - map(attribute='item') | list) - | default(required_roles, True) }}" + with_items: "{{ required_roles }}" - name: Append user overridden roles set_fact: @@ -140,7 +87,7 @@ dest: "{{ item.path | default(role_path_default) }}/{{ item.name | default(item.src | basename) }}" version: "{{ item.version | default('master') }}" refspec: "{{ item.refspec | default(omit) }}" - depth: "{{ item.depth | default(role_clone_default_depth| default(omit)) }}" + depth: "{{ item.depth | default(role_clone_default_depth | default(omit)) }}" update: true force: true with_items: "{{ clone_roles }}" @@ -155,8 +102,8 @@ required_roles: "{{ lookup('file', role_file) | from_yaml }}" role_file: "{{ playbook_dir }}/../ansible-role-requirements.yml" role_path_default: '/etc/ansible/roles' - user_roles: "{{ lookup('file', user_role_path, errors='ignore')|default([], true) | from_yaml }}" - user_role_path: "{{ config_dir ~ '/' ~ (user_role_file|default('')) }}" + user_roles: "{{ lookup('file', user_role_path, errors='ignore') | default([], true) | from_yaml }}" + user_role_path: "{{ config_dir ~ '/' ~ (user_role_file | default('')) }}" git_clone_retries: 2 git_clone_retry_delay: 5 role_clone_default_depth: 20 diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 3b7332fc19..dcaeba290d 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -21,6 +21,7 @@ pre-run: - zuul.d/playbooks/pre-gate-cleanup.yml - zuul.d/playbooks/pre-gate-scenario.yml + - zuul.d/playbooks/pre-osa-requirements.yml - zuul.d/playbooks/pre-osa-aio.yml run: zuul.d/playbooks/run.yml post-run: diff --git a/zuul.d/playbooks/pre-osa-aio.yml b/zuul.d/playbooks/pre-osa-aio.yml index 5500cb5467..e26ecf5714 100644 --- a/zuul.d/playbooks/pre-osa-aio.yml +++ b/zuul.d/playbooks/pre-osa-aio.yml @@ -27,6 +27,7 @@ name: safe.directory scope: global value: "{{ _zuul_src_path }}/opendev.org/openstack/openstack-ansible" + - name: Run bootstrap-ansible script become: yes become_user: root @@ -43,6 +44,7 @@ when: - "'upgrade' not in action" - osa_pre_run_bootstrap | default(True) + - name: Run bootstrap-aio script become: yes become_user: root diff --git a/zuul.d/playbooks/pre-osa-requirements.yml b/zuul.d/playbooks/pre-osa-requirements.yml new file mode 100644 index 0000000000..a02e51dad7 --- /dev/null +++ b/zuul.d/playbooks/pre-osa-requirements.yml @@ -0,0 +1,132 @@ +--- +# Copyright 2025, Cleura AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare the OSA collection/role requirements + hosts: "{{ pre_osa_host | default('all[0]') }}" + vars: + _zuul_src_path: "{{ lookup('env', 'ZUUL_SRC_PATH') | default(lookup('env', 'HOME') ~ '/src') }}" + tasks: + - name: Loading osa-gate-scenario vars + include_vars: + file: "{{ zuul.executor.work_root | default('') }}/osa-gate-scenario.yml" + when: load_zuul_vars | default(true) | bool + + - name: Ensure required directory exists + become: true + file: + path: "{{ item }}" + state: directory + mode: '0755' + with_items: + - /etc/openstack_deploy + - /etc/ansible/roles + + - name: Prepare required roles + when: + - action != 'shastest' + - "'upgrade' not in action" + block: + - name: Check the Zuul src dir for cloned roles + stat: + path: "{{ _zuul_src_path }}/{{ item.src.split('/')[-3:] | join('/') }}" + get_attributes: no + get_checksum: no + get_mime: no + register: zuul_roles + when: + - item.scm == "git" or item.scm is undefined + with_items: "{{ lookup('file', playbook_dir | dirname | dirname ~ '/ansible-role-requirements.yml') | from_yaml }}" + + - name: Remove target directory if required + file: + path: "/etc/ansible/roles/{{ item.item.name | default(item.item.src | basename) }}" + state: absent + with_items: "{{ (lookup('env', 'DROP_ROLE_DIRS') | bool is true) | ternary(zuul_roles.results | selectattr('stat.exists'), []) }}" + + - name: Link the Zuul provided roles + become: true + file: + src: "{{ _zuul_src_path }}/{{ item.item.src.split('/')[-3:] | join('/') }}" + dest: "/etc/ansible/roles/{{ item.item.name | default(item.item.src | basename) }}" + state: link + owner: root + group: root + with_items: "{{ zuul_roles.results | selectattr('stat.exists') }}" + + # NOTE(mnaser): We need to make sure that all the roles + # are checked out by Zuul so we hard fail + # if any roles are not. + - name: Fail if any roles were not cloned + fail: + msg: | + The following roles were not cloned automatically by Zuul, + make sure that they're included in required-projects {{ uncloned_roles | join(',') }} + when: uncloned_roles | length > 0 + vars: + uncloned_roles: "{{ zuul_roles.results | rejectattr('stat.exists') + | map(attribute='item') + | map(attribute='src') + | select('match', 'opendev.org') + | list + }}" + + - name: Prevent prepared roles from being cloned + become: true + copy: + content: |- + {% set ignored_roles = [] %} + {% for role in zuul_roles.results | selectattr('stat.exists') | map(attribute='item') %} + {% set _ = ignored_roles.append({'name': role['name']}) %} + {% endfor %} + {{ ignored_roles | to_nice_yaml }} + dest: /etc/openstack_deploy/user-role-requirements.yml + mode: "0644" + backup: true + + - name: Prepare required collections + when: + - action != 'shastest' + - "'upgrade' not in action" + block: + - name: Check the Zuul src dir for cloned collections + stat: + path: "{{ _zuul_src_path }}/{{ item.source.split('/')[2:] | join('/') | split('#') | first }}" + get_attributes: no + get_checksum: no + get_mime: no + register: zuul_collections + with_items: "{{ (lookup('file', playbook_dir | dirname | dirname ~ '/ansible-collection-requirements.yml') | from_yaml).collections }}" + + - name: Copy content into user-collection-requirements + become: true + vars: + content_var: + collections: |- + {% set collections = [] %} + {% for result in zuul_collections.results %} + {% if result.stat.exists %} + {% set _ = collections.append({ + 'name': result.item.name, + 'source': _zuul_src_path ~ '/' ~ result.item.source.split('/')[2:] | join('/') | replace('#', ''), + 'type': 'dir' + }) + %} + {% endif %} + {% endfor %} + {{ collections }} + copy: + content: "{{ content_var | to_nice_yaml }}" + dest: "/etc/openstack_deploy/user-collection-requirements.yml" + mode: "0644"