From 3b040f58f72cf0f92475e0eb194fed545955cf0d Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Thu, 20 Jun 2019 12:32:35 +0100 Subject: [PATCH] Move non-nested perfload shell commands to script The script was embedded in the playbook, which leads to some pain with regard to editing and reviewing as well as manual testing. The disadvantage of doing this is that it can make jobs somewhat less portable between projects, but in this case that's not really an issue. There are further improvements that can made to remove duplication between the nested and non-nested versions of these jobs. This change will make it easier for those changes to be made as people have time. Change-Id: Ia6795ef15a03429c19e66ed6d297f62da72cc052 --- gate/perfload-runner.sh | 109 ++++++++++++++++++++++++++++++++ playbooks/perfload.yaml | 137 +--------------------------------------- 2 files changed, 111 insertions(+), 135 deletions(-) create mode 100755 gate/perfload-runner.sh diff --git a/gate/perfload-runner.sh b/gate/perfload-runner.sh new file mode 100755 index 000000000..2a7b1a77b --- /dev/null +++ b/gate/perfload-runner.sh @@ -0,0 +1,109 @@ +#!/bin/bash -x + +WORK_DIR=$1 + +# Do some performance related information gathering for placement. +EXPLANATION=" +This output combines output from placeload with timing information +gathered via curl. The placeload output is the current maximum +microversion of placement followed by an encoded representation of +what it has done. Lowercase 'r', 'i', 'a', and 't' indicate successful +creation of a resource provider and setting inventory, aggregates, and +traits on that resource provider. + +If there are upper case versions of any of those letters, a failure +happened for a single request. The letter will be followed by the +HTTP status code and the resource provider uuid. These can be used +to find the relevant entry in logs/screen-placement-api.txt.gz. + +Note that placeload does not exit with an error code when this +happens. It merely reports and moves on. Under correct circumstances +the right output is a long string of 4000 characters containing +'r', 'i', 'a', 't' in random order (because async). + +After that are three aggregate uuids, timing information for the +placeload run, and then timing information for two identical curl +requests for allocation candidates. + +If no timed requests are present it means that the expected number +of resource providers were not created. At this time, only resource +providers are counted, not whether they have the correct inventory, +aggregates, or traits. + +" + +# This aggregate uuid is a static value in placeload. +AGGREGATE="14a5c8a3-5a99-4e8f-88be-00d85fcb1c17" +TRAIT="HW_CPU_X86_AVX2" +PLACEMENT_QUERY="resources=VCPU:1,DISK_GB:10,MEMORY_MB:256&member_of=${AGGREGATE}&required=${TRAIT}" +PLACEMENT_URL="http://localhost:8000" + + +LOG=placement-perf.txt +LOG_DEST=${WORK_DIR}/logs +COUNT=1000 + +trap "sudo cp -p $LOG $LOG_DEST" EXIT + +function time_candidates { + ( + echo "##### TIMING GET /allocation_candidates?${PLACEMENT_QUERY} twice" + time curl -s -H 'x-auth-token: admin' -H 'openstack-api-version: placement latest' "${PLACEMENT_URL}/allocation_candidates?${PLACEMENT_QUERY}" > /dev/null + time curl -s -H 'x-auth-token: admin' -H 'openstack-api-version: placement latest' "${PLACEMENT_URL}/allocation_candidates?${PLACEMENT_QUERY}" > /dev/null + ) 2>&1 | tee -a $LOG +} + +function write_allocation { + # Take the first allocation request and send it back as a well-formed allocation + curl -s -H 'x-auth-token: admin' -H 'openstack-api-version: placement latest' "${PLACEMENT_URL}/allocation_candidates?${PLACEMENT_QUERY}&limit=5" \ + | jq --arg proj $(uuidgen) --arg user $(uuidgen) '.allocation_requests[0] + {consumer_generation: null, project_id: $proj, user_id: $user}' \ + | curl -s -H 'x-auth-token: admin' -H 'content-type: application/json' -H 'openstack-api-version: placement latest' \ + -X PUT -d @- "${PLACEMENT_URL}/allocations/$(uuidgen)" +} + +function load_candidates { + time_candidates + for iter in {1..99}; do + echo "##### Writing allocation ${iter}" | tee -a $LOG + write_allocation + time_candidates + done +} + +function check_placement { + local rp_count + local code + code=0 + + python -m virtualenv -p python3 .placeload + . .placeload/bin/activate + + # install placeload + pip install 'placeload==0.3.0' + + set +x + # load with placeload + ( + echo "$EXPLANATION" + # preheat the aggregates to avoid https://bugs.launchpad.net/nova/+bug/1804453 + placeload $PLACEMENT_URL 10 + echo "##### TIMING placeload creating $COUNT resource providers with inventory, aggregates and traits." + time placeload $PLACEMENT_URL $COUNT + ) 2>&1 | tee -a $LOG + rp_count=$(curl -H 'x-auth-token: admin' ${PLACEMENT_URL}/resource_providers |json_pp|grep -c '"name"') + # Skip curl and note if we failed to create the required number of rps + if [[ $rp_count -ge $COUNT ]]; then + load_candidates + else + ( + echo "Unable to create expected number of resource providers. Expected: ${COUNT}, Got: $rp_count" + echo "See job-output.txt.gz and logs/screen-placement-api.txt.gz for additional detail." + ) | tee -a $LOG + code=1 + fi + set -x + deactivate + exit $code +} + +check_placement diff --git a/playbooks/perfload.yaml b/playbooks/perfload.yaml index 802152ce2..0b494670c 100644 --- a/playbooks/perfload.yaml +++ b/playbooks/perfload.yaml @@ -1,6 +1,5 @@ - hosts: all tasks: - # Borrowed from devstack tasks but we want it early. - name: Ensure {{ ansible_user_dir }}/logs exists become: true file: @@ -12,142 +11,10 @@ chdir: "{{ ansible_user_dir }}/src/opendev.org/openstack/placement" shell: executable: /bin/bash - cmd: | - set -x - # TODO(cdent): Presumably ansible can do this, perhaps with 'package'. - # create database - sudo debconf-set-selections < /dev/null - time curl -s -H 'x-auth-token: admin' -H 'openstack-api-version: placement latest' "${PLACEMENT_URL}/allocation_candidates?${PLACEMENT_QUERY}" > /dev/null - ) 2>&1 | tee -a $LOG - } - - function write_allocation { - # Take the first allocation request and send it back as a well-formed allocation - curl -s -H 'x-auth-token: admin' -H 'openstack-api-version: placement latest' "${PLACEMENT_URL}/allocation_candidates?${PLACEMENT_QUERY}&limit=5" \ - | jq --arg proj $(uuidgen) --arg user $(uuidgen) '.allocation_requests[0] + {consumer_generation: null, project_id: $proj, user_id: $user}' \ - | curl -s -H 'x-auth-token: admin' -H 'content-type: application/json' -H 'openstack-api-version: placement latest' \ - -X PUT -d @- "${PLACEMENT_URL}/allocations/$(uuidgen)" - } - - function load_candidates { - time_candidates - for iter in {1..99}; do - echo "##### Writing allocation ${iter}" | tee -a $LOG - write_allocation - time_candidates - done - } - - function check_placement { - local rp_count - local code - code=0 - - python -m virtualenv -p python3 .placeload - . .placeload/bin/activate - - # install placeload - pip install 'placeload==0.3.0' - - set +x - # load with placeload - ( - echo "$EXPLANATION" - # preheat the aggregates to avoid https://bugs.launchpad.net/nova/+bug/1804453 - placeload $PLACEMENT_URL 10 - echo "##### TIMING placeload creating $COUNT resource providers with inventory, aggregates and traits." - time placeload $PLACEMENT_URL $COUNT - ) 2>&1 | tee -a $LOG - rp_count=$(curl -H 'x-auth-token: admin' ${PLACEMENT_URL}/resource_providers |json_pp|grep -c '"name"') - # Skip curl and note if we failed to create the required number of rps - if [[ $rp_count -ge $COUNT ]]; then - load_candidates - else - ( - echo "Unable to create expected number of resource providers. Expected: ${COUNT}, Got: $rp_count" - echo "See job-output.txt.gz and logs/screen-placement-api.txt.gz for additional detail." - ) | tee -a $LOG - code=1 - fi - set -x - deactivate - exit $code - } - - check_placement + cmd: gate/perfload-runner.sh {{ ansible_user_dir }}