openstack/openstack-ansible
Code Issues Proposed changes

Move zuul preparation for role/collection bootstrap

Browse Source 
At the moment bootstrap-ansible.sh playbooks are bloated with
irrelevant for end-user and applicable only in CI tasks that
ensure respect for depends-on in Zuul.

Here it's proposed to move all CI-related activities from user
facing scripts to Zull pre-tasks separately, which should
remove confusing tasks from the bootstrap process and slightly
speed-up its execution.

Change-Id: I2f94bbf00719b23714bcf855127a0d22170f9064
This commit is contained in:
Dmitriy Rabotyagov
2025-01-13 19:29:45 +01:00
parent 29d2291e55
commit bc517cfcc5
6 changed files with 179 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
scripts/gate-check-commit.sh
View File

@@ -256,7 +256,6 @@ if [[ "${ACTION}" =~ "upgrade" ]]; then
# We need this as in stein we were deploying custom
# /etc/openstack_deploy/env.d/aio_metal.yml for metal installs
export SKIP_CUSTOM_ENVD_CHECK=true
export DROP_ROLE_DIRS=true
# Export ZUUL_SRC_PATH only when integrated repo folder exists. Based on that
# we make an assumption about if we're in CI or not
@@ -264,6 +263,12 @@ if [[ "${ACTION}" =~ "upgrade" ]]; then
export ZUUL_SRC_PATH="/home/zuul/src"
# Doing symlinking here, as bootstrap role won't be called
ln -s $ZUUL_SRC_PATH /openstack/src
# Run a zuul playbook to place use zuul-prepared roles/requirements to
# respect defined Depends-On on N. action=deploy is set as otherwise tasks will be skipped.
export DROP_ROLE_DIRS=true
ansible-playbook /home/zuul/src/opendev.org/openstack/openstack-ansible/zuul.d/playbooks/pre-osa-requirements.yml \
-e pre_osa_host=localhost -e load_zuul_vars=False -e action=deploy
unset DROP_ROLE_DIRS
fi
# Update AIO config files for certain scenarios
# for item in "${SCENARIOS_WITH_CONFIG_UPDATE[@]}"; do

82
scripts/get-ansible-collection-requirements.yml
View File

@@ -29,73 +29,55 @@
set_fact:
user_collection_names: "{{ user_collections.collections | default([]) | map(attribute='name') | list }}"
- name: Get Zuul cloned repos path in CI
set_fact:
zuul_src_path: "{{ lookup('env', 'ZUUL_SRC_PATH') }}"
- name: Generate a list of required collections excluding user overridden collections
set_fact:
galaxy_collections_list : "{{ (galaxy_collections_list | default([])) + [ item ] }}"
galaxy_collections_list: "{{ (galaxy_collections_list | default([])) + [item] }}"
when:
- item.name not in user_collection_names
with_items: "{{ required_collections.collections }}"
- name: Append user collections to filtered required collections
set_fact:
galaxy_collections_list: "{{ (galaxy_collections_list | default([])) + [ item ] }}"
galaxy_collections_list: "{{ (galaxy_collections_list | default([])) + [item] }}"
with_items: "{{ user_collections.collections }}"
when:
- user_collections.collections is defined
- "'source' in item"
- name: Check the Zuul src dir for cloned collections
stat:
path: "{{ zuul_src_path }}/{{ item.source.split('/')[2:] | join('/') | split('#') | first }}"
get_attributes: no
get_checksum: no
get_mime: no
register: zuul_collections
with_items: "{{ galaxy_collections_list }}"
- name: Installing resulting collection list
block:
- name: Create temporary file for galaxy collection requirements
tempfile:
register: collection_requirements_tmpfile
- name: Override paths for zuul hosted collections in CI
vars:
zuul_item:
name: "{{ item.item.name }}"
source: "{{ zuul_src_path }}/{{ item.item.source.split('/')[2:] | join('/') | replace('#', '') }}"
type: "dir"
set_fact:
galaxy_collections_list_ci: "{{ galaxy_collections_list_ci | default([]) + [(zuul_src_path and item.stat.exists) | ternary(zuul_item, item.item)] }}"
with_items: "{{ zuul_collections.results }}"
- name: Copy content into galaxy collection requirements temporary file
vars:
content_var:
collections: "{{ galaxy_collections_list }}"
copy:
content: "{{ content_var | to_nice_yaml }}"
dest: "{{ collection_requirements_tmpfile.path }}"
mode: "0644"
- name: Create temporary file for galaxy collection requirements
tempfile:
register: collection_requirements_tmpfile
- name: Install collection requirements with ansible galaxy
command: >
/opt/ansible-runtime/bin/ansible-galaxy collection install --force
-r "{{ collection_requirements_tmpfile.path }}"
-p "{{ collection_path_default }}"
register: collection_install
until: collection_install is success
retries: 5
delay: 2
- name: Copy content into galaxy collection requirements temporary file
vars:
content_var:
collections: "{{ galaxy_collections_list_ci }}"
copy:
content: "{{ content_var | to_nice_yaml }}"
dest: "{{ collection_requirements_tmpfile.path }}"
- name: Show collection install output
debug:
msg: "{{ collection_install.stdout.split('\n') }}"
- name: Install collection requirements with ansible galaxy
command: >
/opt/ansible-runtime/bin/ansible-galaxy collection install --force
-r "{{ collection_requirements_tmpfile.path }}"
-p "{{ collection_path_default }}"
register: collection_install
until: collection_install is success
retries: 5
delay: 2
- name: Show collection install output
debug: msg="{{ collection_install.stdout.split('\n') }}"
- name: Clean up temporary file
file:
path: "{{ collection_requirements_tmpfile.path }}"
state: absent
always:
- name: Clean up temporary file
file:
path: "{{ collection_requirements_tmpfile.path }}"
state: absent
vars:
collection_file: "{{ playbook_dir }}/../ansible-collection-requirements.yml"

65
scripts/get-ansible-role-requirements.yml
View File

@@ -50,56 +50,7 @@
file:
path: "{{ role_path_default }}"
state: directory
- name: Use Zuul provided sources in Zuul environment
block:
- name: Set Zuul sources path
set_fact:
zuul_src_path: "{{ lookup('env', 'ZUUL_SRC_PATH') }}"
- name: Check the Zuul src dir for cloned roles
stat:
path: "{{ zuul_src_path }}/{{ item.src.split('/')[-3:] | join('/') }}"
get_attributes: no
get_checksum: no
get_mime: no
register: zuul_roles
when:
- item.scm == "git" or item.scm is undefined
with_items: "{{ required_roles }}"
- name: Link the Zuul provided roles
file:
src: "{{ zuul_src_path }}/{{ item.item.src.split('/')[-3:] | join('/') }}"
dest: "{{ item.item.path | default(role_path_default) }}/{{ item.item.name | default(item.item.src | basename) }}"
state: link
owner: root
group: root
with_items: "{{ zuul_roles.results
| selectattr('stat.exists')
| list }}"
# NOTE(mnaser): We need to make sure that all the roles
# are checked out by Zuul so we hard fail
# if any roles are not.
- name: Fail if any roles were not cloned
fail:
msg: |
The following roles were not cloned automatically by Zuul,
make sure that they're included in required-projects {{ uncloned_roles|join(',') }}
when: uncloned_roles | length > 0
vars:
uncloned_roles: "{{ zuul_roles.results | rejectattr('stat.exists')
| map(attribute='item')
| map(attribute='src')
| select('match', 'opendev.org')
| list }}"
- name: Ensure overrides directory exists
file:
path: "{{ config_dir }}"
state: directory
when:
- "lookup('env', 'ZUUL_SRC_PATH') != ''"
- "lookup('env', 'UPGRADE_TARGET_BRANCH') == ''"
mode: "0755"
- name: Generate a list of user overridden roles
set_fact:
@@ -107,15 +58,11 @@
- name: Generate a list of roles excluding user overridden roles
set_fact:
clone_roles: "{{ (clone_roles | default([])) + [ item ] }}"
clone_roles: "{{ (clone_roles | default([])) + [item] }}"
when:
- item.scm == "git" or item.scm is undefined
- item.name not in user_overridden_roles
with_items: "{{ (zuul_roles.results | default([]) |
selectattr('stat', 'defined') |
rejectattr('stat.exists') |
map(attribute='item') | list)
| default(required_roles, True) }}"
with_items: "{{ required_roles }}"
- name: Append user overridden roles
set_fact:
@@ -140,7 +87,7 @@
dest: "{{ item.path | default(role_path_default) }}/{{ item.name | default(item.src | basename) }}"
version: "{{ item.version | default('master') }}"
refspec: "{{ item.refspec | default(omit) }}"
depth: "{{ item.depth | default(role_clone_default_depth| default(omit)) }}"
depth: "{{ item.depth | default(role_clone_default_depth | default(omit)) }}"
update: true
force: true
with_items: "{{ clone_roles }}"
@@ -155,8 +102,8 @@
required_roles: "{{ lookup('file', role_file) | from_yaml }}"
role_file: "{{ playbook_dir }}/../ansible-role-requirements.yml"
role_path_default: '/etc/ansible/roles'
user_roles: "{{ lookup('file', user_role_path, errors='ignore')|default([], true) | from_yaml }}"
user_role_path: "{{ config_dir ~ '/' ~ (user_role_file|default('')) }}"
user_roles: "{{ lookup('file', user_role_path, errors='ignore') | default([], true) | from_yaml }}"
user_role_path: "{{ config_dir ~ '/' ~ (user_role_file | default('')) }}"
git_clone_retries: 2
git_clone_retry_delay: 5
role_clone_default_depth: 20

1
zuul.d/jobs.yaml
View File

@@ -21,6 +21,7 @@
pre-run:
- zuul.d/playbooks/pre-gate-cleanup.yml
- zuul.d/playbooks/pre-gate-scenario.yml
- zuul.d/playbooks/pre-osa-requirements.yml
- zuul.d/playbooks/pre-osa-aio.yml
run: zuul.d/playbooks/run.yml
post-run:

2
zuul.d/playbooks/pre-osa-aio.yml
View File

@@ -27,6 +27,7 @@
name: safe.directory
scope: global
value: "{{ _zuul_src_path }}/opendev.org/openstack/openstack-ansible"
- name: Run bootstrap-ansible script
become: yes
become_user: root
@@ -43,6 +44,7 @@
when:
- "'upgrade' not in action"
- osa_pre_run_bootstrap | default(True)
- name: Run bootstrap-aio script
become: yes
become_user: root

132
zuul.d/playbooks/pre-osa-requirements.yml Normal file
View File

@@ -0,0 +1,132 @@
---
# Copyright 2025, Cleura AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Prepare the OSA collection/role requirements
hosts: "{{ pre_osa_host | default('all[0]') }}"
vars:
_zuul_src_path: "{{ lookup('env', 'ZUUL_SRC_PATH') | default(lookup('env', 'HOME') ~ '/src') }}"
tasks:
- name: Loading osa-gate-scenario vars
include_vars:
file: "{{ zuul.executor.work_root | default('') }}/osa-gate-scenario.yml"
when: load_zuul_vars | default(true) | bool
- name: Ensure required directory exists
become: true
file:
path: "{{ item }}"
state: directory
mode: '0755'
with_items:
- /etc/openstack_deploy
- /etc/ansible/roles
- name: Prepare required roles
when:
- action != 'shastest'
- "'upgrade' not in action"
block:
- name: Check the Zuul src dir for cloned roles
stat:
path: "{{ _zuul_src_path }}/{{ item.src.split('/')[-3:] | join('/') }}"
get_attributes: no
get_checksum: no
get_mime: no
register: zuul_roles
when:
- item.scm == "git" or item.scm is undefined
with_items: "{{ lookup('file', playbook_dir | dirname | dirname ~ '/ansible-role-requirements.yml') | from_yaml }}"
- name: Remove target directory if required
file:
path: "/etc/ansible/roles/{{ item.item.name | default(item.item.src | basename) }}"
state: absent
with_items: "{{ (lookup('env', 'DROP_ROLE_DIRS') | bool is true) | ternary(zuul_roles.results | selectattr('stat.exists'), []) }}"
- name: Link the Zuul provided roles
become: true
file:
src: "{{ _zuul_src_path }}/{{ item.item.src.split('/')[-3:] | join('/') }}"
dest: "/etc/ansible/roles/{{ item.item.name | default(item.item.src | basename) }}"
state: link
owner: root
group: root
with_items: "{{ zuul_roles.results | selectattr('stat.exists') }}"
# NOTE(mnaser): We need to make sure that all the roles
# are checked out by Zuul so we hard fail
# if any roles are not.
- name: Fail if any roles were not cloned
fail:
msg: |
The following roles were not cloned automatically by Zuul,
make sure that they're included in required-projects {{ uncloned_roles | join(',') }}
when: uncloned_roles | length > 0
vars:
uncloned_roles: "{{ zuul_roles.results | rejectattr('stat.exists')
| map(attribute='item')
| map(attribute='src')
| select('match', 'opendev.org')
| list
}}"
- name: Prevent prepared roles from being cloned
become: true
copy:
content: |-
{% set ignored_roles = [] %}
{% for role in zuul_roles.results | selectattr('stat.exists') | map(attribute='item') %}
{% set _ = ignored_roles.append({'name': role['name']}) %}
{% endfor %}
{{ ignored_roles | to_nice_yaml }}
dest: /etc/openstack_deploy/user-role-requirements.yml
mode: "0644"
backup: true
- name: Prepare required collections
when:
- action != 'shastest'
- "'upgrade' not in action"
block:
- name: Check the Zuul src dir for cloned collections
stat:
path: "{{ _zuul_src_path }}/{{ item.source.split('/')[2:] | join('/') | split('#') | first }}"
get_attributes: no
get_checksum: no
get_mime: no
register: zuul_collections
with_items: "{{ (lookup('file', playbook_dir | dirname | dirname ~ '/ansible-collection-requirements.yml') | from_yaml).collections }}"
- name: Copy content into user-collection-requirements
become: true
vars:
content_var:
collections: |-
{% set collections = [] %}
{% for result in zuul_collections.results %}
{% if result.stat.exists %}
{% set _ = collections.append({
'name': result.item.name,
'source': _zuul_src_path ~ '/' ~ result.item.source.split('/')[2:] | join('/') | replace('#', ''),
'type': 'dir'
})
%}
{% endif %}
{% endfor %}
{{ collections }}
copy:
content: "{{ content_var | to_nice_yaml }}"
dest: "/etc/openstack_deploy/user-collection-requirements.yml"
mode: "0644"