openstack/devstack
Code Issues Proposed changes

Wait for hypervisor-stats to reflect ironic nodes

Browse Source 
When enrolling nodes into Ironic, poll nova's hypervisor-stats until
the hypervisor count reflects the number of enrolled nodes.  This
eliminates a race where devstack completes and an instance is spawned
before the first post-enrollment periodic task ticks on the Nova side,
which has recently started popping up in the gate.

Change-Id: Ib3d8005e0094ee8af2d5fcb65aca6cd92736da90
Closes-bug: #1398128
This commit is contained in:
Adam Gandelman 2014-12-01 11:24:37 -08:00
parent 3b1647310d
commit c78e467109
1 changed files with 21 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
lib/ironic
View File

@ -500,9 +500,23 @@ function create_bridge_and_vms {
create_ovs_taps create_ovs_taps
} }
function wait_for_nova_resources {
# After nodes have been enrolled, we need to wait for n-cpu's periodic
# task populate the resource tracker with available nodes. Wait for 2
# minutes before timing out.
local expected_count=$1
echo_summary "Waiting 2 minutes for Nova resource tracker to pick up $expected_count Ironic nodes"
for i in $(seq 1 120); do
if [ $(nova hypervisor-stats | grep " count " | get_field 2) -ge $expected_count ]; then
return 0
fi
sleep 1
done
die $LINENO "Nova hypervisor-stats did not register at least $expected_count nodes"
}
function enroll_nodes { function enroll_nodes {
local chassis_id=$(ironic chassis-create -d "ironic test chassis" | grep " uuid " | get_field 2) local chassis_id=$(ironic chassis-create -d "ironic test chassis" | grep " uuid " | get_field 2)
local idx=0
if [[ "$IRONIC_DEPLOY_DRIVER" == "pxe_ssh" ]] ; then if [[ "$IRONIC_DEPLOY_DRIVER" == "pxe_ssh" ]] ; then
local _IRONIC_DEPLOY_KERNEL_KEY=pxe_deploy_kernel local _IRONIC_DEPLOY_KERNEL_KEY=pxe_deploy_kernel
@ -536,6 +550,7 @@ function enroll_nodes {
fi fi
fi fi
local total_nodes=0
while read hardware_info; do while read hardware_info; do
if ! is_ironic_hardware; then if ! is_ironic_hardware; then
local mac_address=$hardware_info local mac_address=$hardware_info
@ -566,7 +581,7 @@ function enroll_nodes {
ironic port-create --address $mac_address --node_uuid $node_id ironic port-create --address $mac_address --node_uuid $node_id
idx=$((idx+1)) total_nodes=$((total_nodes+1))
done < $ironic_hwinfo_file done < $ironic_hwinfo_file
# create the nova flavor # create the nova flavor
@ -581,6 +596,10 @@ function enroll_nodes {
# from the flavor after the completion of # from the flavor after the completion of
# https://blueprints.launchpad.net/ironic/+spec/add-node-instance-info # https://blueprints.launchpad.net/ironic/+spec/add-node-instance-info
nova flavor-key baremetal set "cpu_arch"="x86_64" "baremetal:deploy_kernel_id"="$IRONIC_DEPLOY_KERNEL_ID" "baremetal:deploy_ramdisk_id"="$IRONIC_DEPLOY_RAMDISK_ID" nova flavor-key baremetal set "cpu_arch"="x86_64" "baremetal:deploy_kernel_id"="$IRONIC_DEPLOY_KERNEL_ID" "baremetal:deploy_ramdisk_id"="$IRONIC_DEPLOY_RAMDISK_ID"
if [ "$VIRT_DRIVER" == "ironic" ]; then
wait_for_nova_resources $total_nodes
fi
} }
function configure_iptables { function configure_iptables {