ironic-inspector/devstack/exercise.sh
Dmitry Tantsur 51f7cfbe8d Increase cleaning timeout for gate to 10 minutes
See for example this failed job:
http://logs.openstack.org/18/250418/2/check/gate-ironic-inspector-dsvm-ipa-nv/a1dc8d3/

Change-Id: I7b52f6754121c2cfcb60b20db1d733d511391efd
2015-11-27 13:50:14 +01:00

231 lines
7.0 KiB
Bash
Executable File

#!/bin/bash
set -eux
INTROSPECTION_SLEEP=${INTROSPECTION_SLEEP:-30}
export IRONIC_API_VERSION=${IRONIC_API_VERSION:-latest}
# Copied from devstack
PRIVATE_NETWORK_NAME=${PRIVATE_NETWORK_NAME:-"private"}
rules_file=$(mktemp)
cat > "$rules_file" << EOM
[
{
"description": "Successful Rule",
"conditions": [
{"op": "ge", "field": "memory_mb", "value": 256},
{"op": "ge", "field": "local_gb", "value": 1}
],
"actions": [
{"action": "set-attribute", "path": "/extra/rule_success",
"value": "yes"}
]
},
{
"description": "Failing Rule",
"conditions": [
{"op": "lt", "field": "memory_mb", "value": 42},
{"op": "eq", "field": "local_gb", "value": 0}
],
"actions": [
{"action": "set-attribute", "path": "/extra/rule_success",
"value": "no"},
{"action": "fail", "message": "This rule should not have run"}
]
}
]
EOM
expected_cpus=$(openstack flavor show baremetal -f value -c vcpus)
expected_memory_mb=$(openstack flavor show baremetal -f value -c ram)
expected_cpu_arch=$(openstack flavor show baremetal -f value -c properties | sed "s/.*cpu_arch='\([^']*\)'.*/\1/")
disk_size=$(openstack flavor show baremetal -f value -c disk)
ephemeral_size=$(openstack flavor show baremetal -f value -c "OS-FLV-EXT-DATA:ephemeral")
expected_local_gb=$(($disk_size + $ephemeral_size))
# FIXME(dtantsur): switch to OSC as soon as `openstack endpoint list` actually
# works on devstack
ironic_url=$(keystone endpoint-get --service baremetal | tail -n +4 | head -n -1 | tr '|' ' ' | awk '{ print $2; }')
if [ -z "$ironic_url" ]; then
echo "Cannot find Ironic URL"
exit 1
fi
# NOTE(dtantsur): it's hard to get JSON field from Ironic client output, using
# HTTP API and JQ instead.
function curl_ir {
local token=$(openstack token issue -f value -c id)
curl -H "X-Auth-Token: $token" -X $1 "$ironic_url/$2"
}
function curl_ins {
local token=$(openstack token issue -f value -c id)
local args=${3:-}
curl -f -H "X-Auth-Token: $token" -X $1 $args "http://127.0.0.1:5050/$2"
}
nodes=$(ironic node-list | tail -n +4 | head -n -1 | tr '|' ' ' | awk '{ print $1; }')
if [ -z "$nodes" ]; then
echo "No nodes found in Ironic"
exit 1
fi
for uuid in $nodes; do
for p in cpus cpu_arch memory_mb local_gb; do
ironic node-update $uuid remove properties/$p > /dev/null || true
done
if ! ironic node-show $uuid | grep provision_state | grep -iq manageable; then
ironic node-set-provision-state $uuid manage
fi
done
openstack baremetal introspection rule purge
openstack baremetal introspection rule import "$rules_file"
for uuid in $nodes; do
ironic node-set-provision-state $uuid inspect
done
current_nodes=$nodes
temp_nodes=
while true; do
sleep $INTROSPECTION_SLEEP
for uuid in $current_nodes; do
finished=$(openstack baremetal introspection status $uuid -f value -c finished)
if [ "$finished" = "True" ]; then
error=$(openstack baremetal introspection status $uuid -f value -c error)
if [ "$error" != "None" ]; then
echo "Introspection for $uuid failed: $error"
exit 1
fi
else
temp_nodes="$temp_nodes $uuid"
fi
done
if [ "$temp_nodes" = "" ]; then
echo "Introspection done"
break
else
current_nodes=$temp_nodes
temp_nodes=
fi
done
openstack baremetal introspection rule purge
function test_swift {
# Basic sanity check of the data stored in Swift
stored_data_json=$(curl_ins GET v1/introspection/$uuid/data)
stored_cpu_arch=$(echo $stored_data_json | jq -r '.cpu_arch')
echo CPU arch for $uuid from stored data: $stored_cpu_arch
if [ "$stored_cpu_arch" != "$expected_cpu_arch" ]; then
echo "The data stored in Swift does not match the expected data."
exit 1
fi
}
function wait_for_provision_state {
local uuid=$1
local expected=$2
local max_attempts=${3:-6}
for attempt in $(seq 1 $max_attempts); do
local current=$(ironic node-show $uuid | grep ' provision_state ' | awk '{ print $4; }')
if [ "$current" != "$expected" ]; then
if [ "$attempt" -eq "$max_attempts" ]; then
echo "Expected provision_state $expected, got $current:"
ironic node-show $uuid
exit 1
fi
else
break
fi
sleep 10
done
}
for uuid in $nodes; do
node_json=$(curl_ir GET v1/nodes/$uuid)
properties=$(echo $node_json | jq '.properties')
echo Properties for $uuid: $properties
if [ "$(echo $properties | jq -r '.cpu_arch')" != "$expected_cpu_arch" ]; then
echo "Expected CPU architecture: $expected_cpu_arch"
exit 1
fi
if [ "$(echo $properties | jq -r '.cpus')" != "$expected_cpus" ]; then
echo "Expected number of CPUS: $expected_cpus"
exit 1
fi
if [ "$(echo $properties | jq -r '.local_gb')" != "$expected_local_gb" ]; then
echo "Expected disk: $expected_local_gb"
exit 1
fi
if [ "$(echo $properties | jq -r '.memory_mb')" != "$expected_memory_mb" ]; then
echo "Expected memory: $expected_memory_mb"
exit 1
fi
extra=$(echo $node_json | jq '.extra')
echo Extra properties for $uuid: $extra
if [ "$(echo $extra | jq -r '.rule_success')" != "yes" ]; then
echo "Rule matching failed"
exit 1
fi
openstack service list | grep swift && test_swift
wait_for_provision_state $uuid manageable
ironic node-set-provision-state $uuid provide
done
# Cleaning kicks in here, we have to wait until it finishes (~ 2 minutes)
for uuid in $nodes; do
wait_for_provision_state $uuid available 60 # 10 minutes for cleaning
done
echo "Wait until nova becomes aware of bare metal instances"
for attempt in {1..24}; do
if [ $(nova hypervisor-stats | grep ' vcpus ' | head -n1 | awk '{ print $4; }') -ge $expected_cpus ]; then
break
elif [ "$attempt" -eq 24 ]; then
echo "Timeout while waiting for nova hypervisor-stats, current:"
nova hypervisor-stats
exit 1
fi
sleep 5
done
echo "Try nova boot for one instance"
image=$(openstack image list --property disk_format=ami -f value -c ID | head -n1)
net_id=$(neutron net-list | egrep "$PRIVATE_NETWORK_NAME"'[^-]' | awk '{ print $2 }')
uuid=$(nova boot --flavor baremetal --nic net-id=$net_id --image $image testing | grep " id " | awk '{ print $4 }')
for attempt in {1..30}; do
status=$(nova show $uuid | grep " status " | awk '{ print $4 }')
if [ "$status" = "ERROR" ]; then
echo "Instance failed to boot"
# Some debug output
nova show $uuid
nova hypervisor-stats
exit 1
elif [ "$status" != "ACTIVE" ]; then
if [ "$attempt" -eq 30 ]; then
echo "Instance didn't become ACTIVE, status is $status"
exit 1
fi
else
break
fi
sleep 30
done
nova delete $uuid
echo "Validation passed"