openstack/ironic
Code Issues Proposed changes

Adds RPC calls for rescue interface

Browse Source 
This commit adds RPC calls for rescue interface.
It also adds transitions to/from the rescue-related states.

Change-Id: I12cc8c3b89588394ff10837f05dd6ad5e9b55ee7
Partial-bug: #1526449
Co-Authored-By: Jay Faulkner <jay@jvf.cc>
Co-Authored-By: Josh Gachnang <josh@pcsforeducation.com>
Co-Authored-By: Jesse J. Cook <jesse.j.cook@member.fsf.org>
Co-Authored-By: Mario Villaplana <mario.villaplana@gmail.com>
Co-Authored-By: Aparna <aparnavtce@gmail.com>
Co-Authored-By: Shivanand Tendulker <stendulker@gmail.com>
This commit is contained in:
Shivanand Tendulker 2017-10-03 14:09:50 -04:00
parent 67d225268e
commit 084da02a32
11 changed files with 1163 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

538
doc/source/images/states.svg
View File

@ -1,338 +1,482 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- Generated by graphviz version 2.36.0 (20140111.2315)
<!-- Generated by graphviz version 2.38.0 (20140413.2041)
-->
<!-- Title: Ironic states Pages: 1 -->
<svg width="1724pt" height="503pt"
viewBox="0.00 0.00 1724.00 502.55" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 498.55)">
<svg width="3138pt" height="777pt"
viewBox="0.00 0.00 3137.57 776.58" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 772.576)">
<title>Ironic states</title>
<polygon fill="white" stroke="none" points="-4,4 -4,-498.55 1720,-498.55 1720,4 -4,4"/>
<polygon fill="white" stroke="none" points="-4,4 -4,-772.576 3133.57,-772.576 3133.57,4 -4,4"/>
<!-- enroll -->
<g id="node1" class="node"><title>enroll</title>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="27" cy="-187.55" rx="27" ry="18"/>
<text text-anchor="middle" x="27" y="-184.75" font-family="Times,serif" font-size="11.00">enroll</text>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="30.6682" cy="-177.576" rx="30.8374" ry="18"/>
<text text-anchor="middle" x="30.6682" y="-174.776" font-family="Times,serif" font-size="11.00">enroll</text>
</g>
<!-- verifying -->
<g id="node2" class="node"><title>verifying</title>
<ellipse fill="none" stroke="black" cx="210" cy="-187.55" rx="33.8507" ry="18"/>
<text text-anchor="middle" x="210" y="-184.75" font-family="Times,serif" font-size="11.00" fill="gray">verifying</text>
<ellipse fill="none" stroke="black" cx="244.227" cy="-177.576" rx="40.7822" ry="18"/>
<text text-anchor="middle" x="244.227" y="-174.776" font-family="Times,serif" font-size="11.00" fill="gray">verifying</text>
</g>
<!-- enroll&#45;&gt;verifying -->
<g id="edge1" class="edge"><title>enroll&#45;&gt;verifying</title>
<path fill="none" stroke="black" d="M54.319,-187.55C83.5522,-187.55 131.193,-187.55 165.889,-187.55"/>
<polygon fill="black" stroke="black" points="166.207,-191.05 176.207,-187.55 166.207,-184.05 166.207,-191.05"/>
<text text-anchor="middle" x="115" y="-190.95" font-family="Times,serif" font-size="12.00">manage (via API)</text>
<path fill="none" stroke="black" d="M61.5308,-177.576C95.6561,-177.576 152.061,-177.576 192.985,-177.576"/>
<polygon fill="black" stroke="black" points="193.033,-181.076 203.033,-177.576 193.033,-174.076 193.033,-181.076"/>
<text text-anchor="middle" x="132.336" y="-180.976" font-family="Times,serif" font-size="12.00">manage (via API)</text>
</g>
<!-- verifying&#45;&gt;enroll -->
<g id="edge13" class="edge"><title>verifying&#45;&gt;enroll</title>
<path fill="none" stroke="black" d="M182.674,-176.984C174.862,-174.341 166.199,-171.881 158,-170.55 120.271,-164.428 109.575,-163.544 72,-170.55 67.7433,-171.344 63.368,-172.535 59.1046,-173.925"/>
<polygon fill="black" stroke="black" points="57.6647,-170.725 49.4683,-177.438 60.0622,-177.301 57.6647,-170.725"/>
<text text-anchor="middle" x="115" y="-173.95" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
<g id="edge17" class="edge"><title>verifying&#45;&gt;enroll</title>
<path fill="none" stroke="black" d="M211.739,-166.561C203.273,-164.089 194.046,-161.828 185.336,-160.576 138.705,-153.87 125.758,-152.546 79.3365,-160.576 74.8317,-161.355 70.189,-162.52 65.6505,-163.881"/>
<polygon fill="black" stroke="black" points="64.3578,-160.622 55.9753,-167.102 66.5689,-167.264 64.3578,-160.622"/>
<text text-anchor="middle" x="132.336" y="-163.976" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g>
<!-- manageable -->
<g id="node3" class="node"><title>manageable</title>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="346" cy="-187.55" rx="42.1875" ry="18"/>
<text text-anchor="middle" x="346" y="-184.75" font-family="Times,serif" font-size="11.00">manageable</text>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="407.239" cy="-177.576" rx="54.2411" ry="18"/>
<text text-anchor="middle" x="407.239" y="-174.776" font-family="Times,serif" font-size="11.00">manageable</text>
</g>
<!-- verifying&#45;&gt;manageable -->
<g id="edge12" class="edge"><title>verifying&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M243.78,-187.55C258.666,-187.55 276.633,-187.55 293.273,-187.55"/>
<polygon fill="black" stroke="black" points="293.422,-191.05 303.422,-187.55 293.422,-184.05 293.422,-191.05"/>
<text text-anchor="middle" x="274" y="-190.95" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
<g id="edge16" class="edge"><title>verifying&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M285.442,-177.576C302.733,-177.576 323.366,-177.576 342.643,-177.576"/>
<polygon fill="black" stroke="black" points="342.879,-181.076 352.879,-177.576 342.879,-174.076 342.879,-181.076"/>
<text text-anchor="middle" x="319.118" y="-180.976" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
</g>
<!-- cleaning -->
<g id="node4" class="node"><title>cleaning</title>
<ellipse fill="none" stroke="black" cx="551" cy="-212.55" rx="32.4445" ry="18"/>
<text text-anchor="middle" x="551" y="-209.75" font-family="Times,serif" font-size="11.00" fill="gray">cleaning</text>
<ellipse fill="none" stroke="black" cx="654.359" cy="-211.576" rx="40.7822" ry="18"/>
<text text-anchor="middle" x="654.359" y="-208.776" font-family="Times,serif" font-size="11.00" fill="gray">cleaning</text>
</g>
<!-- manageable&#45;&gt;cleaning -->
<g id="edge2" class="edge"><title>manageable&#45;&gt;cleaning</title>
<path fill="none" stroke="black" d="M368.881,-202.811C379.556,-209.344 392.898,-216.226 406,-219.55 440.464,-228.294 481.019,-225.179 510.294,-220.673"/>
<polygon fill="black" stroke="black" points="511.247,-224.063 520.537,-218.969 510.098,-217.158 511.247,-224.063"/>
<text text-anchor="middle" x="448" y="-227.95" font-family="Times,serif" font-size="12.00">provide (via API)</text>
<path fill="none" stroke="black" d="M435.697,-193.121C448.535,-199.541 464.287,-206.25 479.359,-209.576 520.843,-218.728 569.253,-218.429 604.464,-216.301"/>
<polygon fill="black" stroke="black" points="604.744,-219.79 614.486,-215.624 604.273,-212.806 604.744,-219.79"/>
<text text-anchor="middle" x="531.359" y="-220.976" font-family="Times,serif" font-size="12.00">provide (via API)</text>
</g>
<!-- manageable&#45;&gt;cleaning -->
<g id="edge3" class="edge"><title>manageable&#45;&gt;cleaning</title>
<path fill="none" stroke="black" d="M387.452,-191.134C416.131,-193.836 455.493,-197.869 490,-202.55 496.387,-203.416 503.138,-204.438 509.718,-205.492"/>
<polygon fill="black" stroke="black" points="509.403,-208.987 519.838,-207.156 510.539,-202.08 509.403,-208.987"/>
<text text-anchor="middle" x="448" y="-205.95" font-family="Times,serif" font-size="12.00">clean (via API)</text>
<path fill="none" stroke="black" d="M461.051,-179.626C495.97,-181.587 542.646,-185.367 583.359,-192.576 592.278,-194.155 601.711,-196.347 610.666,-198.682"/>
<polygon fill="black" stroke="black" points="609.952,-202.115 620.519,-201.355 611.784,-195.359 609.952,-202.115"/>
<text text-anchor="middle" x="531.359" y="-195.976" font-family="Times,serif" font-size="12.00">clean (via API)</text>
</g>
<!-- inspecting -->
<g id="node5" class="node"><title>inspecting</title>
<ellipse fill="none" stroke="black" cx="551" cy="-25.55" rx="37.0671" ry="18"/>
<text text-anchor="middle" x="551" y="-22.75" font-family="Times,serif" font-size="11.00" fill="gray">inspecting</text>
<ellipse fill="none" stroke="black" cx="654.359" cy="-28.5757" rx="46.8089" ry="18"/>
<text text-anchor="middle" x="654.359" y="-25.7757" font-family="Times,serif" font-size="11.00" fill="gray">inspecting</text>
</g>
<!-- manageable&#45;&gt;inspecting -->
<g id="edge4" class="edge"><title>manageable&#45;&gt;inspecting</title>
<path fill="none" stroke="black" d="M349.471,-169.433C354.494,-140.014 368.771,-82.1703 406,-52.55 420.855,-40.731 467.343,-33.5154 503.525,-29.5531"/>
<polygon fill="black" stroke="black" points="504.314,-32.99 513.898,-28.4735 503.589,-26.0276 504.314,-32.99"/>
<text text-anchor="middle" x="448" y="-55.95" font-family="Times,serif" font-size="12.00">inspect (via API)</text>
<path fill="none" stroke="black" d="M413.247,-159.638C421.876,-132.333 442.291,-80.8522 479.359,-55.5757 498.182,-42.7405 554.071,-35.6986 597.483,-32.0549"/>
<polygon fill="black" stroke="black" points="597.968,-35.5275 607.658,-31.2429 597.411,-28.5496 597.968,-35.5275"/>
<text text-anchor="middle" x="531.359" y="-58.9757" font-family="Times,serif" font-size="12.00">inspect (via API)</text>
</g>
<!-- adopting -->
<g id="node6" class="node"><title>adopting</title>
<ellipse fill="none" stroke="black" cx="551" cy="-412.55" rx="32.4445" ry="18"/>
<text text-anchor="middle" x="551" y="-409.75" font-family="Times,serif" font-size="11.00" fill="gray">adopting</text>
<ellipse fill="none" stroke="black" cx="654.359" cy="-429.576" rx="42.1875" ry="18"/>
<text text-anchor="middle" x="654.359" y="-426.776" font-family="Times,serif" font-size="11.00" fill="gray">adopting</text>
</g>
<!-- manageable&#45;&gt;adopting -->
<g id="edge5" class="edge"><title>manageable&#45;&gt;adopting</title>
<path fill="none" stroke="black" d="M347.291,-205.559C348.896,-242.554 357.935,-327.494 406,-373.55 433.219,-399.631 476.034,-408.662 507.913,-411.6"/>
<polygon fill="black" stroke="black" points="508.033,-415.119 518.267,-412.379 508.557,-408.139 508.033,-415.119"/>
<text text-anchor="middle" x="448" y="-411.95" font-family="Times,serif" font-size="12.00">adopt (via API)</text>
<path fill="none" stroke="black" d="M408.623,-195.573C410.526,-236.7 421.202,-338.055 479.359,-390.576 512.541,-420.541 563.662,-429.109 601.92,-430.885"/>
<polygon fill="black" stroke="black" points="602.238,-434.397 612.344,-431.218 602.461,-427.401 602.238,-434.397"/>
<text text-anchor="middle" x="531.359" y="-431.976" font-family="Times,serif" font-size="12.00">adopt (via API)</text>
</g>
<!-- cleaning&#45;&gt;manageable -->
<g id="edge26" class="edge"><title>cleaning&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M531.764,-198.035C520.454,-189.983 505.219,-180.765 490,-176.55 458.156,-167.73 420.901,-171.01 392.401,-176.215"/>
<polygon fill="black" stroke="black" points="391.458,-172.833 382.324,-178.201 392.812,-179.701 391.458,-172.833"/>
<text text-anchor="middle" x="448" y="-179.95" font-family="Times,serif" font-size="12.00" fill="gray">manage</text>
<g id="edge30" class="edge"><title>cleaning&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M635.626,-195.284C622.333,-184.236 603.103,-170.589 583.359,-164.576 543.013,-152.287 495.131,-157.077 459.642,-164.075"/>
<polygon fill="black" stroke="black" points="458.583,-160.721 449.513,-166.198 460.019,-167.572 458.583,-160.721"/>
<text text-anchor="middle" x="531.359" y="-167.976" font-family="Times,serif" font-size="12.00" fill="gray">manage</text>
</g>
<!-- available -->
<g id="node7" class="node"><title>available</title>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="763" cy="-313.55" rx="34.054" ry="18"/>
<text text-anchor="middle" x="763" y="-310.75" font-family="Times,serif" font-size="11.00">available</text>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="907.689" cy="-327.576" rx="42.8909" ry="18"/>
<text text-anchor="middle" x="907.689" y="-324.776" font-family="Times,serif" font-size="11.00">available</text>
</g>
<!-- cleaning&#45;&gt;available -->
<g id="edge23" class="edge"><title>cleaning&#45;&gt;available</title>
<path fill="none" stroke="black" d="M566.869,-228.52C578.227,-239.83 594.807,-254.535 612,-263.55 647.041,-281.923 660.573,-273.719 698,-286.55 707.652,-289.859 717.915,-293.93 727.33,-297.898"/>
<polygon fill="black" stroke="black" points="726.205,-301.224 736.775,-301.959 728.971,-294.793 726.205,-301.224"/>
<text text-anchor="middle" x="655" y="-289.95" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
<g id="edge27" class="edge"><title>cleaning&#45;&gt;available</title>
<path fill="none" stroke="black" d="M668.753,-228.655C681.624,-243.86 702.454,-265.51 725.359,-277.576 768.011,-300.043 785.015,-287.304 831.359,-300.576 842.825,-303.859 855.073,-307.977 866.288,-312"/>
<polygon fill="black" stroke="black" points="865.239,-315.343 875.833,-315.488 867.642,-308.768 865.239,-315.343"/>
<text text-anchor="middle" x="778.359" y="-303.976" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
</g>
<!-- clean failed -->
<g id="node14" class="node"><title>clean failed</title>
<ellipse fill="none" stroke="black" cx="964" cy="-214.55" rx="41.4846" ry="18"/>
<text text-anchor="middle" x="964" y="-211.75" font-family="Times,serif" font-size="11.00" fill="red">clean failed</text>
<g id="node17" class="node"><title>clean failed</title>
<ellipse fill="none" stroke="black" cx="1147.73" cy="-177.576" rx="51.931" ry="18"/>
<text text-anchor="middle" x="1147.73" y="-174.776" font-family="Times,serif" font-size="11.00" fill="red">clean failed</text>
</g>
<!-- cleaning&#45;&gt;clean failed -->
<g id="edge24" class="edge"><title>cleaning&#45;&gt;clean failed</title>
<path fill="none" stroke="black" d="M580.467,-220.482C590.368,-222.895 601.598,-225.251 612,-226.55 740.778,-242.631 775.06,-242.281 904,-227.55 908.518,-227.034 913.199,-226.32 917.857,-225.489"/>
<polygon fill="black" stroke="black" points="918.803,-228.87 927.946,-223.517 917.46,-222 918.803,-228.87"/>
<text text-anchor="middle" x="763" y="-241.95" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
<g id="edge28" class="edge"><title>cleaning&#45;&gt;clean failed</title>
<path fill="none" stroke="black" d="M695.26,-212.735C770.566,-214.293 938.605,-214.761 1078.02,-193.576 1083.84,-192.691 1089.89,-191.576 1095.89,-190.346"/>
<polygon fill="black" stroke="black" points="1096.84,-193.722 1105.88,-188.191 1095.36,-186.879 1096.84,-193.722"/>
<text text-anchor="middle" x="907.689" y="-215.976" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g>
<!-- clean wait -->
<g id="node15" class="node"><title>clean wait</title>
<ellipse fill="none" stroke="black" cx="763" cy="-201.55" rx="37.7689" ry="18"/>
<text text-anchor="middle" x="763" y="-198.75" font-family="Times,serif" font-size="11.00" fill="gray">clean wait</text>
<g id="node18" class="node"><title>clean wait</title>
<ellipse fill="none" stroke="black" cx="907.689" cy="-171.576" rx="46.1069" ry="18"/>
<text text-anchor="middle" x="907.689" y="-168.776" font-family="Times,serif" font-size="11.00" fill="gray">clean wait</text>
</g>
<!-- cleaning&#45;&gt;clean wait -->
<g id="edge25" class="edge"><title>cleaning&#45;&gt;clean wait</title>
<path fill="none" stroke="black" d="M583.439,-210.903C618.279,-209.078 674.834,-206.116 714.953,-204.014"/>
<polygon fill="black" stroke="black" points="715.228,-207.505 725.031,-203.486 714.862,-200.514 715.228,-207.505"/>
<text text-anchor="middle" x="655" y="-212.95" font-family="Times,serif" font-size="12.00" fill="gray">wait</text>
<g id="edge29" class="edge"><title>cleaning&#45;&gt;clean wait</title>
<path fill="none" stroke="black" d="M692.008,-204.514C702.743,-202.515 714.509,-200.39 725.359,-198.576 768.359,-191.384 817.258,-184.158 853.432,-178.994"/>
<polygon fill="black" stroke="black" points="854.347,-182.399 863.755,-177.527 853.362,-175.469 854.347,-182.399"/>
<text text-anchor="middle" x="778.359" y="-201.976" font-family="Times,serif" font-size="12.00" fill="gray">wait</text>
</g>
<!-- inspecting&#45;&gt;manageable -->
<g id="edge33" class="edge"><title>inspecting&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M522.516,-13.7533C491.262,-2.36256 440.155,9.88802 406,-14.55 359.666,-47.703 349.303,-119.445 347.246,-159.343"/>
<polygon fill="black" stroke="black" points="343.743,-159.336 346.868,-169.459 350.739,-159.597 343.743,-159.336"/>
<text text-anchor="middle" x="448" y="-17.95" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
<g id="edge37" class="edge"><title>inspecting&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M621.586,-15.6814C584.332,-2.81929 522.236,11.3578 479.359,-16.5757 434.152,-46.0274 417.472,-111.443 411.474,-149.202"/>
<polygon fill="black" stroke="black" points="407.95,-149.118 409.993,-159.514 414.879,-150.113 407.95,-149.118"/>
<text text-anchor="middle" x="531.359" y="-19.9757" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
</g>
<!-- inspect failed -->
<g id="node16" class="node"><title>inspect failed</title>
<ellipse fill="none" stroke="black" cx="763" cy="-53.55" rx="46.1069" ry="18"/>
<text text-anchor="middle" x="763" y="-50.75" font-family="Times,serif" font-size="11.00" fill="red">inspect failed</text>
<g id="node19" class="node"><title>inspect failed</title>
<ellipse fill="none" stroke="black" cx="907.689" cy="-56.5757" rx="58.16" ry="18"/>
<text text-anchor="middle" x="907.689" y="-53.7757" font-family="Times,serif" font-size="11.00" fill="red">inspect failed</text>
</g>
<!-- inspecting&#45;&gt;inspect failed -->
<g id="edge34" class="edge"><title>inspecting&#45;&gt;inspect failed</title>
<path fill="none" stroke="black" d="M582.246,-35.4105C591.683,-38.1369 602.191,-40.8192 612,-42.55 643.032,-48.0255 678.14,-50.7831 706.585,-52.1685"/>
<polygon fill="black" stroke="black" points="706.598,-55.6724 716.744,-52.6193 706.909,-48.6793 706.598,-55.6724"/>
<text text-anchor="middle" x="655" y="-55.95" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
<g id="edge38" class="edge"><title>inspecting&#45;&gt;inspect failed</title>
<path fill="none" stroke="black" d="M692.898,-38.9911C703.334,-41.5308 714.724,-43.9746 725.359,-45.5757 762.667,-51.1919 804.702,-53.9613 838.901,-55.3181"/>
<polygon fill="black" stroke="black" points="839.128,-58.8286 849.249,-55.6954 839.383,-51.8332 839.128,-58.8286"/>
<text text-anchor="middle" x="778.359" y="-58.9757" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g>
<!-- active -->
<g id="node9" class="node"><title>active</title>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="1172" cy="-410.55" rx="27" ry="18"/>
<text text-anchor="middle" x="1172" y="-407.75" font-family="Times,serif" font-size="11.00">active</text>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="1396.38" cy="-471.576" rx="31.0408" ry="18"/>
<text text-anchor="middle" x="1396.38" y="-468.776" font-family="Times,serif" font-size="11.00">active</text>
</g>
<!-- adopting&#45;&gt;active -->
<g id="edge37" class="edge"><title>adopting&#45;&gt;active</title>
<path fill="none" stroke="black" d="M582.779,-408.379C592.145,-407.267 602.478,-406.195 612,-405.55 658.125,-402.423 669.776,-404.37 716,-403.55 852.885,-401.123 887.098,-399.661 1024,-398.55 1060.44,-398.254 1069.71,-395.228 1106,-398.55 1115.82,-399.449 1126.37,-401.096 1136.05,-402.888"/>
<polygon fill="black" stroke="black" points="1135.66,-406.377 1146.15,-404.861 1137,-399.508 1135.66,-406.377"/>
<text text-anchor="middle" x="866" y="-403.95" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
<g id="edge41" class="edge"><title>adopting&#45;&gt;active</title>
<path fill="none" stroke="black" d="M693.34,-422.627C767.471,-410.046 936.558,-386.16 1078.02,-401.576 1180.93,-412.791 1299.35,-443.889 1358.19,-460.607"/>
<polygon fill="black" stroke="black" points="1357.54,-464.064 1368.12,-463.455 1359.47,-457.335 1357.54,-464.064"/>
<text text-anchor="middle" x="1031.02" y="-404.976" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
</g>
<!-- adopt failed -->
<g id="node17" class="node"><title>adopt failed</title>
<ellipse fill="none" stroke="black" cx="763" cy="-430.55" rx="41.4846" ry="18"/>
<text text-anchor="middle" x="763" y="-427.75" font-family="Times,serif" font-size="11.00" fill="red">adopt failed</text>
<g id="node20" class="node"><title>adopt failed</title>
<ellipse fill="none" stroke="black" cx="907.689" cy="-447.576" rx="53.5381" ry="18"/>
<text text-anchor="middle" x="907.689" y="-444.776" font-family="Times,serif" font-size="11.00" fill="red">adopt failed</text>
</g>
<!-- adopting&#45;&gt;adopt failed -->
<g id="edge38" class="edge"><title>adopting&#45;&gt;adopt failed</title>
<path fill="none" stroke="black" d="M578.165,-422.754C588.531,-426.286 600.662,-429.798 612,-431.55 645.152,-436.674 682.919,-436.388 712.263,-434.839"/>
<polygon fill="black" stroke="black" points="712.495,-438.332 722.27,-434.246 712.081,-431.344 712.495,-438.332"/>
<text text-anchor="middle" x="655" y="-438.95" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
<g id="edge42" class="edge"><title>adopting&#45;&gt;adopt failed</title>
<path fill="none" stroke="black" d="M688.107,-440.39C699.734,-443.714 713.002,-446.928 725.359,-448.576 764.98,-453.857 809.903,-453.654 845.123,-452.107"/>
<polygon fill="black" stroke="black" points="845.437,-455.596 855.255,-451.614 845.097,-448.604 845.437,-455.596"/>
<text text-anchor="middle" x="778.359" y="-455.976" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g>
<!-- available&#45;&gt;manageable -->
<g id="edge7" class="edge"><title>available&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M728.535,-313.4C662.605,-311.739 513.031,-301.112 406,-241.55 392.136,-233.835 378.894,-222.248 368.48,-211.784"/>
<polygon fill="black" stroke="black" points="370.878,-209.227 361.43,-204.434 365.826,-214.073 370.878,-209.227"/>
<text text-anchor="middle" x="551" y="-303.95" font-family="Times,serif" font-size="12.00">manage (via API)</text>
<path fill="none" stroke="black" d="M864.906,-327.11C827.911,-326.046 772.608,-322.854 725.359,-313.576 612.678,-291.449 580.507,-288.941 479.359,-234.576 462.738,-225.642 446.17,-212.663 433.211,-201.346"/>
<polygon fill="black" stroke="black" points="435.425,-198.63 425.641,-194.565 430.755,-203.844 435.425,-198.63"/>
<text text-anchor="middle" x="654.359" y="-312.976" font-family="Times,serif" font-size="12.00">manage (via API)</text>
</g>
<!-- deploying -->
<g id="node8" class="node"><title>deploying</title>
<ellipse fill="none" stroke="black" cx="964" cy="-320.55" rx="35.4579" ry="18"/>
<text text-anchor="middle" x="964" y="-317.75" font-family="Times,serif" font-size="11.00" fill="gray">deploying</text>
<ellipse fill="none" stroke="black" cx="1147.73" cy="-610.576" rx="44.498" ry="18"/>
<text text-anchor="middle" x="1147.73" y="-607.776" font-family="Times,serif" font-size="11.00" fill="gray">deploying</text>
</g>
<!-- available&#45;&gt;deploying -->
<g id="edge6" class="edge"><title>available&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M797.308,-314.722C830.44,-315.887 881.492,-317.683 918.17,-318.973"/>
<polygon fill="black" stroke="black" points="918.464,-322.486 928.581,-319.339 918.71,-315.49 918.464,-322.486"/>
<text text-anchor="middle" x="866" y="-320.95" font-family="Times,serif" font-size="12.00">active (via API)</text>
<path fill="none" stroke="black" d="M922.792,-344.732C937.883,-363.063 962.553,-392.948 984.019,-418.576 1034.62,-478.989 1094.69,-549.578 1125.76,-586.017"/>
<polygon fill="black" stroke="black" points="1123.15,-588.344 1132.3,-593.68 1128.47,-583.801 1123.15,-588.344"/>
<text text-anchor="middle" x="1031.02" y="-530.976" font-family="Times,serif" font-size="12.00">active (via API)</text>
</g>
<!-- deploying&#45;&gt;active -->
<g id="edge16" class="edge"><title>deploying&#45;&gt;active</title>
<path fill="none" stroke="black" d="M984.128,-335.45C995.3,-343.648 1009.93,-353.547 1024,-360.55 1060.97,-378.95 1106.33,-393.232 1136.8,-401.737"/>
<polygon fill="black" stroke="black" points="1135.96,-405.136 1146.53,-404.397 1137.81,-398.383 1135.96,-405.136"/>
<text text-anchor="middle" x="1065" y="-393.95" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
<g id="edge20" class="edge"><title>deploying&#45;&gt;active</title>
<path fill="none" stroke="black" d="M1152.42,-592.617C1159.43,-563.575 1177.68,-506.995 1217.45,-481.576 1259.15,-454.921 1318.21,-457.672 1356.65,-463.488"/>
<polygon fill="black" stroke="black" points="1356.48,-467.007 1366.91,-465.187 1357.62,-460.101 1356.48,-467.007"/>
<text text-anchor="middle" x="1268.45" y="-484.976" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
</g>
<!-- deploy failed -->
<g id="node12" class="node"><title>deploy failed</title>
<ellipse fill="none" stroke="black" cx="1317" cy="-283.55" rx="44.498" ry="18"/>
<text text-anchor="middle" x="1317" y="-280.75" font-family="Times,serif" font-size="11.00" fill="red">deploy failed</text>
<g id="node15" class="node"><title>deploy failed</title>
<ellipse fill="none" stroke="black" cx="1646.24" cy="-643.576" rx="55.8489" ry="18"/>
<text text-anchor="middle" x="1646.24" y="-640.776" font-family="Times,serif" font-size="11.00" fill="red">deploy failed</text>
</g>
<!-- deploying&#45;&gt;deploy failed -->
<g id="edge14" class="edge"><title>deploying&#45;&gt;deploy failed</title>
<path fill="none" stroke="black" d="M996.054,-312.65C1005.02,-310.659 1014.85,-308.748 1024,-307.55 1125.45,-294.27 1152.58,-311.095 1254,-297.55 1258.81,-296.908 1263.79,-296.074 1268.75,-295.133"/>
<polygon fill="black" stroke="black" points="1269.72,-298.506 1278.82,-293.078 1268.32,-291.648 1269.72,-298.506"/>
<text text-anchor="middle" x="1172" y="-305.95" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
<g id="edge18" class="edge"><title>deploying&#45;&gt;deploy failed</title>
<path fill="none" stroke="black" d="M1186.09,-619.902C1259.85,-637.168 1429.42,-671.168 1572.31,-657.576 1577.85,-657.049 1583.58,-656.303 1589.3,-655.426"/>
<polygon fill="black" stroke="black" points="1590.23,-658.819 1599.52,-653.724 1589.08,-651.914 1590.23,-658.819"/>
<text text-anchor="middle" x="1396.38" y="-661.976" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g>
<!-- wait call&#45;back -->
<g id="node13" class="node"><title>wait call&#45;back</title>
<ellipse fill="none" stroke="black" cx="1172" cy="-348.55" rx="48.2143" ry="18"/>
<text text-anchor="middle" x="1172" y="-345.75" font-family="Times,serif" font-size="11.00" fill="gray">wait call&#45;back</text>
<g id="node16" class="node"><title>wait call&#45;back</title>
<ellipse fill="none" stroke="black" cx="1396.38" cy="-708.576" rx="58.8623" ry="18"/>
<text text-anchor="middle" x="1396.38" y="-705.776" font-family="Times,serif" font-size="11.00" fill="gray">wait call&#45;back</text>
</g>
<!-- deploying&#45;&gt;wait call&#45;back -->
<g id="edge15" class="edge"><title>deploying&#45;&gt;wait call&#45;back</title>
<path fill="none" stroke="black" d="M995.764,-328.698C1004.81,-330.863 1014.76,-333.021 1024,-334.55 1053.6,-339.448 1086.87,-342.794 1114.25,-344.983"/>
<polygon fill="black" stroke="black" points="1114.2,-348.489 1124.44,-345.766 1114.74,-341.51 1114.2,-348.489"/>
<text text-anchor="middle" x="1065" y="-346.95" font-family="Times,serif" font-size="12.00" fill="gray">wait</text>
<g id="edge19" class="edge"><title>deploying&#45;&gt;wait call&#45;back</title>
<path fill="none" stroke="black" d="M1155.7,-628.531C1165.59,-651.196 1186.04,-689.197 1217.45,-705.576 1254.13,-724.707 1301.12,-724.644 1337.6,-720.233"/>
<polygon fill="black" stroke="black" points="1338.15,-723.691 1347.59,-718.886 1337.22,-716.754 1338.15,-723.691"/>
<text text-anchor="middle" x="1268.45" y="-725.976" font-family="Times,serif" font-size="12.00" fill="gray">wait</text>
</g>
<!-- active&#45;&gt;deploying -->
<g id="edge8" class="edge"><title>active&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M1150.34,-421.324C1120.29,-435.36 1063.77,-455.453 1024,-431.55 994.275,-413.687 978.719,-375.231 971.196,-348.631"/>
<polygon fill="black" stroke="black" points="974.525,-347.53 968.61,-338.74 967.753,-349.3 974.525,-347.53"/>
<text text-anchor="middle" x="1065" y="-445.95" font-family="Times,serif" font-size="12.00">rebuild (via API)</text>
<path fill="none" stroke="black" d="M1369.18,-480.474C1321.31,-496.783 1224.01,-530.202 1217.45,-534.576 1197.28,-548.019 1179.12,-568.681 1166.54,-585.069"/>
<polygon fill="black" stroke="black" points="1163.65,-583.079 1160.49,-593.189 1169.27,-587.262 1163.65,-583.079"/>
<text text-anchor="middle" x="1268.45" y="-537.976" font-family="Times,serif" font-size="12.00">rebuild (via API)</text>
</g>
<!-- deleting -->
<g id="node10" class="node"><title>deleting</title>
<ellipse fill="none" stroke="black" cx="1512" cy="-348.55" rx="31.0408" ry="18"/>
<text text-anchor="middle" x="1512" y="-345.75" font-family="Times,serif" font-size="11.00" fill="gray">deleting</text>
<ellipse fill="none" stroke="black" cx="2893.96" cy="-534.576" rx="39.1741" ry="18"/>
<text text-anchor="middle" x="2893.96" y="-531.776" font-family="Times,serif" font-size="11.00" fill="gray">deleting</text>
</g>
<!-- active&#45;&gt;deleting -->
<g id="edge9" class="edge"><title>active&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M1198.7,-407.281C1249.41,-400.613 1365.74,-384.222 1462,-362.55 1466.08,-361.632 1470.32,-360.576 1474.53,-359.464"/>
<polygon fill="black" stroke="black" points="1475.47,-362.834 1484.18,-356.807 1473.61,-356.085 1475.47,-362.834"/>
<text text-anchor="middle" x="1317" y="-399.95" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
<path fill="none" stroke="black" d="M1419.57,-483.817C1470.69,-511.225 1601.44,-576.895 1720.16,-599.576 1776.66,-610.37 2263.87,-644.811 2836.87,-550.576 2841.74,-549.776 2846.78,-548.701 2851.76,-547.486"/>
<polygon fill="black" stroke="black" points="2853.03,-550.77 2861.8,-544.835 2851.24,-544.002 2853.03,-550.77"/>
<text text-anchor="middle" x="2121.48" y="-617.976" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g>
<!-- rescuing -->
<g id="node11" class="node"><title>rescuing</title>
<ellipse fill="none" stroke="black" cx="1646.24" cy="-466.576" rx="40.7822" ry="18"/>
<text text-anchor="middle" x="1646.24" y="-463.776" font-family="Times,serif" font-size="11.00" fill="gray">rescuing</text>
</g>
<!-- active&#45;&gt;rescuing -->
<g id="edge10" class="edge"><title>active&#45;&gt;rescuing</title>
<path fill="none" stroke="black" d="M1427.77,-470.963C1469.41,-470.123 1544.72,-468.603 1594.86,-467.592"/>
<polygon fill="black" stroke="black" points="1595.23,-471.085 1605.15,-467.384 1595.09,-464.087 1595.23,-471.085"/>
<text text-anchor="middle" x="1522.81" y="-472.976" font-family="Times,serif" font-size="12.00">rescue (via API)</text>
</g>
<!-- deleting&#45;&gt;cleaning -->
<g id="edge32" class="edge"><title>deleting&#45;&gt;cleaning</title>
<path fill="none" stroke="black" d="M1503.74,-331.082C1482.69,-283.573 1416.8,-155.55 1318,-155.55 762,-155.55 762,-155.55 762,-155.55 694.339,-155.55 676.135,-159.993 612,-181.55 601.998,-184.912 591.612,-189.742 582.35,-194.591"/>
<polygon fill="black" stroke="black" points="580.63,-191.542 573.517,-199.394 583.974,-197.692 580.63,-191.542"/>
<text text-anchor="middle" x="1065" y="-158.95" font-family="Times,serif" font-size="12.00" fill="gray">clean</text>
<g id="edge36" class="edge"><title>deleting&#45;&gt;cleaning</title>
<path fill="none" stroke="black" d="M2859.68,-525.664C2822.97,-514.079 2764.88,-490.228 2732.87,-448.576 2711.72,-421.051 2740.49,-395.997 2714.87,-372.576 2403.2,-87.6439 2192.95,-266.576 1770.66,-266.576 906.689,-266.576 906.689,-266.576 906.689,-266.576 831.054,-266.576 745.514,-242.511 696.265,-226.218"/>
<polygon fill="black" stroke="black" points="697.212,-222.844 686.618,-222.972 694.979,-229.478 697.212,-222.844"/>
<text text-anchor="middle" x="1769.66" y="-269.976" font-family="Times,serif" font-size="12.00" fill="gray">clean</text>
</g>
<!-- error -->
<g id="node11" class="node"><title>error</title>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="1689" cy="-384.55" rx="27" ry="18"/>
<text text-anchor="middle" x="1689" y="-381.75" font-family="Times,serif" font-size="11.00" fill="red">error</text>
<g id="node12" class="node"><title>error</title>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="3101.31" cy="-570.576" rx="28.0277" ry="18"/>
<text text-anchor="middle" x="3101.31" y="-567.776" font-family="Times,serif" font-size="11.00" fill="red">error</text>
</g>
<!-- deleting&#45;&gt;error -->
<g id="edge31" class="edge"><title>deleting&#45;&gt;error</title>
<path fill="none" stroke="black" d="M1541.12,-341.962C1568.37,-336.931 1610.39,-332.8 1644,-345.55 1652.92,-348.933 1661.19,-355.042 1668.09,-361.437"/>
<polygon fill="black" stroke="black" points="1666,-364.298 1675.53,-368.928 1670.97,-359.366 1666,-364.298"/>
<text text-anchor="middle" x="1603" y="-348.95" font-family="Times,serif" font-size="12.00" fill="gray">error</text>
<g id="edge35" class="edge"><title>deleting&#45;&gt;error</title>
<path fill="none" stroke="black" d="M2929.59,-526.815C2962.87,-520.981 3014,-516.333 3055.04,-531.576 3063.99,-534.898 3072.35,-540.869 3079.39,-547.157"/>
<polygon fill="black" stroke="black" points="3077.4,-550.094 3087.01,-554.543 3082.27,-545.068 3077.4,-550.094"/>
<text text-anchor="middle" x="3003.04" y="-534.976" font-family="Times,serif" font-size="12.00" fill="gray">error</text>
</g>
<!-- rescue -->
<g id="node13" class="node"><title>rescue</title>
<ellipse fill="none" stroke="black" stroke-width="1.7" cx="2000.48" cy="-558.576" rx="33.8507" ry="18"/>
<text text-anchor="middle" x="2000.48" y="-555.776" font-family="Times,serif" font-size="11.00">rescue</text>
</g>
<!-- rescuing&#45;&gt;rescue -->
<g id="edge45" class="edge"><title>rescuing&#45;&gt;rescue</title>
<path fill="none" stroke="black" d="M1664.32,-482.786C1678.23,-494.988 1698.97,-511.024 1720.16,-519.576 1762.18,-536.535 1889.46,-549.359 1956.74,-555.161"/>
<polygon fill="black" stroke="black" points="1956.74,-558.673 1967,-556.031 1957.33,-551.698 1956.74,-558.673"/>
<text text-anchor="middle" x="1769.66" y="-542.976" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
</g>
<!-- rescue wait -->
<g id="node21" class="node"><title>rescue wait</title>
<ellipse fill="none" stroke="black" cx="1887.07" cy="-410.576" rx="49.8222" ry="18"/>
<text text-anchor="middle" x="1887.07" y="-407.776" font-family="Times,serif" font-size="11.00" fill="gray">rescue wait</text>
</g>
<!-- rescuing&#45;&gt;rescue wait -->
<g id="edge46" class="edge"><title>rescuing&#45;&gt;rescue wait</title>
<path fill="none" stroke="black" d="M1675.14,-453.824C1688.53,-448.139 1704.93,-441.79 1720.16,-437.576 1755.69,-427.745 1796.56,-421.042 1828.84,-416.774"/>
<polygon fill="black" stroke="black" points="1829.53,-420.214 1839,-415.472 1828.64,-413.271 1829.53,-420.214"/>
<text text-anchor="middle" x="1769.66" y="-440.976" font-family="Times,serif" font-size="12.00" fill="gray">wait</text>
</g>
<!-- rescue failed -->
<g id="node22" class="node"><title>rescue failed</title>
<ellipse fill="none" stroke="black" cx="2121.48" cy="-365.576" rx="55.8489" ry="18"/>
<text text-anchor="middle" x="2121.48" y="-362.776" font-family="Times,serif" font-size="11.00" fill="red">rescue failed</text>
</g>
<!-- rescuing&#45;&gt;rescue failed -->
<g id="edge47" class="edge"><title>rescuing&#45;&gt;rescue failed</title>
<path fill="none" stroke="black" d="M1686.95,-468.766C1760.76,-471.356 1922.76,-470.014 2045.98,-418.576 2063.21,-411.385 2080.33,-399.617 2093.8,-388.995"/>
<polygon fill="black" stroke="black" points="2096.14,-391.604 2101.69,-382.58 2091.73,-386.173 2096.14,-391.604"/>
<text text-anchor="middle" x="1887.07" y="-467.976" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g>
<!-- error&#45;&gt;deploying -->
<g id="edge10" class="edge"><title>error&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M1673.16,-399.706C1644.78,-426.988 1579.69,-481.55 1513,-481.55 1171,-481.55 1171,-481.55 1171,-481.55 1104.94,-481.55 1076.23,-499.999 1024,-459.55 989.177,-432.581 974.65,-381.143 968.777,-348.968"/>
<polygon fill="black" stroke="black" points="972.19,-348.151 967.101,-338.858 965.284,-349.296 972.19,-348.151"/>
<text text-anchor="middle" x="1317" y="-484.95" font-family="Times,serif" font-size="12.00">rebuild (via API)</text>
<g id="edge11" class="edge"><title>error&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M3091.88,-587.648C3067.95,-633.447 2994.77,-755.576 2894.96,-755.576 1395.38,-755.576 1395.38,-755.576 1395.38,-755.576 1315.98,-755.576 1283.62,-783.454 1217.45,-739.576 1182.45,-716.369 1163.99,-669.259 1155.3,-638.847"/>
<polygon fill="black" stroke="black" points="1158.6,-637.609 1152.63,-628.853 1151.83,-639.418 1158.6,-637.609"/>
<text text-anchor="middle" x="2121.48" y="-758.976" font-family="Times,serif" font-size="12.00">rebuild (via API)</text>
</g>
<!-- error&#45;&gt;deleting -->
<g id="edge11" class="edge"><title>error&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M1662.26,-380.869C1636.68,-377.021 1596.41,-370.447 1562,-362.55 1557.93,-361.615 1553.69,-360.548 1549.48,-359.43"/>
<polygon fill="black" stroke="black" points="1550.4,-356.052 1539.83,-356.766 1548.53,-362.8 1550.4,-356.052"/>
<text text-anchor="middle" x="1603" y="-380.95" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
<g id="edge12" class="edge"><title>error&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M3073.48,-567.428C3043.49,-563.724 2993.57,-556.996 2951.04,-548.576 2946.84,-547.742 2942.47,-546.797 2938.11,-545.8"/>
<polygon fill="black" stroke="black" points="2938.59,-542.316 2928.05,-543.409 2936.97,-549.127 2938.59,-542.316"/>
<text text-anchor="middle" x="3003.04" y="-567.976" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g>
<!-- rescue&#45;&gt;deleting -->
<g id="edge14" class="edge"><title>rescue&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M2034.27,-557.693C2169.44,-554.054 2683.92,-540.203 2844.67,-535.876"/>
<polygon fill="black" stroke="black" points="2844.84,-539.373 2854.74,-535.605 2844.65,-532.375 2844.84,-539.373"/>
<text text-anchor="middle" x="2381.98" y="-552.976" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g>
<!-- rescue&#45;&gt;rescuing -->
<g id="edge13" class="edge"><title>rescue&#45;&gt;rescuing</title>
<path fill="none" stroke="black" d="M1970.3,-566.677C1917.34,-579.707 1803.23,-599.999 1720.16,-559.576 1692.07,-545.907 1671.6,-515.602 1659.62,-493.284"/>
<polygon fill="black" stroke="black" points="1662.68,-491.585 1655.01,-484.28 1656.45,-494.776 1662.68,-491.585"/>
<text text-anchor="middle" x="1769.66" y="-585.976" font-family="Times,serif" font-size="12.00">rescue (via API)</text>
</g>
<!-- unrescuing -->
<g id="node14" class="node"><title>unrescuing</title>
<ellipse fill="none" stroke="black" cx="2381.98" cy="-365.576" rx="49.8222" ry="18"/>
<text text-anchor="middle" x="2381.98" y="-362.776" font-family="Times,serif" font-size="11.00" fill="gray">unrescuing</text>
</g>
<!-- rescue&#45;&gt;unrescuing -->
<g id="edge15" class="edge"><title>rescue&#45;&gt;unrescuing</title>
<path fill="none" stroke="black" d="M2009.83,-541.024C2020,-521.413 2038.98,-490.005 2063.98,-471.576 2107.06,-439.815 2252.65,-398.788 2330.65,-378.34"/>
<polygon fill="black" stroke="black" points="2331.94,-381.621 2340.74,-375.713 2330.18,-374.847 2331.94,-381.621"/>
<text text-anchor="middle" x="2121.48" y="-474.976" font-family="Times,serif" font-size="12.00">unrescue (via API)</text>
</g>
<!-- unrescuing&#45;&gt;active -->
<g id="edge55" class="edge"><title>unrescuing&#45;&gt;active</title>
<path fill="none" stroke="black" d="M2345.07,-353.416C2295.58,-337.769 2203.34,-312.576 2122.48,-312.576 1645.24,-312.576 1645.24,-312.576 1645.24,-312.576 1546.57,-312.576 1455.81,-402.936 1417.1,-447.419"/>
<polygon fill="black" stroke="black" points="1414.33,-445.282 1410.49,-455.159 1419.65,-449.828 1414.33,-445.282"/>
<text text-anchor="middle" x="1887.07" y="-315.976" font-family="Times,serif" font-size="12.00" fill="gray">done</text>
</g>
<!-- unrescue failed -->
<g id="node23" class="node"><title>unrescue failed</title>
<ellipse fill="none" stroke="black" cx="2649.93" cy="-399.576" rx="64.889" ry="18"/>
<text text-anchor="middle" x="2649.93" y="-396.776" font-family="Times,serif" font-size="11.00" fill="red">unrescue failed</text>
</g>
<!-- unrescuing&#45;&gt;unrescue failed -->
<g id="edge56" class="edge"><title>unrescuing&#45;&gt;unrescue failed</title>
<path fill="none" stroke="black" d="M2425.96,-374.222C2434.56,-375.797 2443.54,-377.332 2451.98,-378.576 2493.34,-384.667 2539.83,-389.708 2577.25,-393.324"/>
<polygon fill="black" stroke="black" points="2577.27,-396.841 2587.56,-394.305 2577.93,-389.873 2577.27,-396.841"/>
<text text-anchor="middle" x="2509.48" y="-394.976" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g>
<!-- deploy failed&#45;&gt;deploying -->
<g id="edge20" class="edge"><title>deploy failed&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M1273.1,-280.157C1215.43,-276.818 1110.18,-275.048 1024,-297.55 1016.3,-299.561 1008.23,-302.276 1000.62,-305.134"/>
<polygon fill="black" stroke="black" points="999.326,-301.882 991.292,-308.789 1001.88,-308.4 999.326,-301.882"/>
<text text-anchor="middle" x="1172" y="-283.95" font-family="Times,serif" font-size="12.00">rebuild (via API)</text>
<g id="edge24" class="edge"><title>deploy failed&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M1591.39,-639.997C1495.7,-633.637 1297.13,-620.439 1201.91,-614.11"/>
<polygon fill="black" stroke="black" points="1202.14,-610.617 1191.93,-613.446 1201.67,-617.602 1202.14,-610.617"/>
<text text-anchor="middle" x="1396.38" y="-633.976" font-family="Times,serif" font-size="12.00">rebuild (via API)</text>
</g>
<!-- deploy failed&#45;&gt;deploying -->
<g id="edge21" class="edge"><title>deploy failed&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M1280.8,-273.016C1262.79,-268.167 1240.42,-262.963 1220,-260.55 1133.13,-250.288 1104.63,-244.616 1024,-278.55 1012.17,-283.53 1000.39,-291.217 990.505,-298.661"/>
<polygon fill="black" stroke="black" points="988.053,-296.136 982.351,-305.066 992.377,-301.641 988.053,-296.136"/>
<text text-anchor="middle" x="1172" y="-263.95" font-family="Times,serif" font-size="12.00">active (via API)</text>
<g id="edge25" class="edge"><title>deploy failed&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M1599.77,-633.378C1590.68,-631.403 1581.2,-629.386 1572.31,-627.576 1520.46,-617.016 1507.99,-610.486 1455.31,-605.576 1403.15,-600.715 1389.83,-604.882 1337.45,-605.576 1291.92,-606.179 1240.22,-607.6 1202.46,-608.774"/>
<polygon fill="black" stroke="black" points="1202.09,-605.284 1192.2,-609.097 1202.31,-612.28 1202.09,-605.284"/>
<text text-anchor="middle" x="1396.38" y="-608.976" font-family="Times,serif" font-size="12.00">active (via API)</text>
</g>
<!-- deploy failed&#45;&gt;deleting -->
<g id="edge22" class="edge"><title>deploy failed&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M1357.34,-291.193C1386.77,-297.561 1427.67,-307.808 1462,-321.55 1468.42,-324.121 1475.06,-327.351 1481.3,-330.675"/>
<polygon fill="black" stroke="black" points="1479.65,-333.758 1490.09,-335.545 1483.04,-327.636 1479.65,-333.758"/>
<text text-anchor="middle" x="1421" y="-324.95" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
<g id="edge26" class="edge"><title>deploy failed&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M1701.77,-646.021C1798.28,-649.644 2005.39,-654.254 2178.98,-636.576 2361.78,-617.96 2402.88,-583.085 2584.98,-558.576 2676.43,-546.268 2784.06,-539.663 2844.82,-536.655"/>
<polygon fill="black" stroke="black" points="2845.06,-540.148 2854.88,-536.169 2844.73,-533.156 2845.06,-540.148"/>
<text text-anchor="middle" x="2254.48" y="-636.976" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g>
<!-- wait call&#45;back&#45;&gt;deploying -->
<g id="edge17" class="edge"><title>wait call&#45;back&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M1145.71,-333.256C1134.06,-327.073 1119.78,-320.668 1106,-317.55 1073.78,-310.26 1036.35,-311.488 1008.16,-314.28"/>
<polygon fill="black" stroke="black" points="1007.78,-310.801 998.217,-315.366 1008.54,-317.759 1007.78,-310.801"/>
<text text-anchor="middle" x="1065" y="-320.95" font-family="Times,serif" font-size="12.00" fill="gray">resume</text>
<g id="edge21" class="edge"><title>wait call&#45;back&#45;&gt;deploying</title>
<path fill="none" stroke="black" d="M1352.15,-696.602C1341.5,-693.814 1330.09,-690.965 1319.45,-688.576 1274.46,-678.474 1259.03,-689.501 1217.45,-669.576 1199.95,-661.192 1183.36,-647.183 1170.86,-634.92"/>
<polygon fill="black" stroke="black" points="1173.15,-632.251 1163.64,-627.575 1168.16,-637.159 1173.15,-632.251"/>
<text text-anchor="middle" x="1268.45" y="-691.976" font-family="Times,serif" font-size="12.00" fill="gray">resume</text>
</g>
<!-- wait call&#45;back&#45;&gt;deleting -->
<g id="edge19" class="edge"><title>wait call&#45;back&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M1220.35,-348.55C1286.99,-348.55 1407.31,-348.55 1470.2,-348.55"/>
<polygon fill="black" stroke="black" points="1470.52,-352.05 1480.52,-348.55 1470.52,-345.05 1470.52,-352.05"/>
<text text-anchor="middle" x="1317" y="-351.95" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
<g id="edge23" class="edge"><title>wait call&#45;back&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M1455.47,-707.842C1505.7,-707.271 1580.26,-706.576 1645.24,-706.576 1645.24,-706.576 1645.24,-706.576 2001.48,-706.576 2056.73,-706.576 2672.7,-596.316 2836.87,-553.576 2842.22,-552.183 2847.79,-550.538 2853.26,-548.808"/>
<polygon fill="black" stroke="black" points="2854.52,-552.08 2862.93,-545.637 2852.33,-545.429 2854.52,-552.08"/>
<text text-anchor="middle" x="2121.48" y="-701.976" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g>
<!-- wait call&#45;back&#45;&gt;deploy failed -->
<g id="edge18" class="edge"><title>wait call&#45;back&#45;&gt;deploy failed</title>
<path fill="none" stroke="black" d="M1203.33,-334.765C1224.94,-324.939 1254.18,-311.651 1277.45,-301.072"/>
<polygon fill="black" stroke="black" points="1279.04,-304.195 1286.7,-296.87 1276.14,-297.822 1279.04,-304.195"/>
<text text-anchor="middle" x="1246" y="-320.95" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
<g id="edge22" class="edge"><title>wait call&#45;back&#45;&gt;deploy failed</title>
<path fill="none" stroke="black" d="M1431.76,-694.121C1444.64,-689.059 1459.5,-683.631 1473.31,-679.576 1510.3,-668.713 1552.69,-659.8 1586.12,-653.56"/>
<polygon fill="black" stroke="black" points="1586.98,-656.959 1596.19,-651.71 1585.72,-650.074 1586.98,-656.959"/>
<text text-anchor="middle" x="1522.81" y="-682.976" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g>
<!-- clean failed&#45;&gt;manageable -->
<g id="edge30" class="edge"><title>clean failed&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M939.234,-199.96C928.717,-193.956 916.028,-187.308 904,-182.55 815.986,-147.736 791.872,-140.649 698,-128.55 585.203,-114.012 453.168,-150.875 387.862,-172.744"/>
<polygon fill="black" stroke="black" points="386.682,-169.448 378.343,-175.984 388.938,-176.074 386.682,-169.448"/>
<text text-anchor="middle" x="655" y="-131.95" font-family="Times,serif" font-size="12.00">manage (via API)</text>
<g id="edge34" class="edge"><title>clean failed&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M1117.01,-162.866C1105.17,-157.702 1091.23,-152.472 1078.02,-149.576 1029.35,-138.907 1015.82,-146.242 966.019,-144.576 749.81,-137.342 691.914,-113.336 479.359,-153.576 470.428,-155.266 461.085,-157.817 452.216,-160.634"/>
<polygon fill="black" stroke="black" points="450.828,-157.408 442.452,-163.896 453.046,-164.047 450.828,-157.408"/>
<text text-anchor="middle" x="778.359" y="-140.976" font-family="Times,serif" font-size="12.00">manage (via API)</text>
</g>
<!-- clean wait&#45;&gt;cleaning -->
<g id="edge29" class="edge"><title>clean wait&#45;&gt;cleaning</title>
<path fill="none" stroke="black" d="M728.957,-193.498C698.382,-187.27 651.785,-180.888 612,-188.55 602.955,-190.292 593.541,-193.385 584.924,-196.783"/>
<polygon fill="black" stroke="black" points="583.465,-193.6 575.596,-200.694 586.172,-200.055 583.465,-193.6"/>
<text text-anchor="middle" x="655" y="-191.95" font-family="Times,serif" font-size="12.00" fill="gray">resume</text>
<g id="edge33" class="edge"><title>clean wait&#45;&gt;cleaning</title>
<path fill="none" stroke="black" d="M866.156,-163.432C828.79,-157.555 772.195,-152.82 725.359,-166.576 709.704,-171.174 694.238,-180.473 681.841,-189.43"/>
<polygon fill="black" stroke="black" points="679.655,-186.695 673.785,-195.515 683.874,-192.281 679.655,-186.695"/>
<text text-anchor="middle" x="778.359" y="-169.976" font-family="Times,serif" font-size="12.00" fill="gray">resume</text>
</g>
<!-- clean wait&#45;&gt;clean failed -->
<g id="edge27" class="edge"><title>clean wait&#45;&gt;clean failed</title>
<path fill="none" stroke="black" d="M800.964,-203.965C832.338,-206.014 877.662,-208.975 912.401,-211.245"/>
<polygon fill="black" stroke="black" points="912.609,-214.765 922.815,-211.925 913.065,-207.78 912.609,-214.765"/>
<text text-anchor="middle" x="866" y="-213.95" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
<g id="edge31" class="edge"><title>clean wait&#45;&gt;clean failed</title>
<path fill="none" stroke="black" d="M954.1,-172.796C963.963,-173.057 974.341,-173.328 984.019,-173.576 1017.65,-174.435 1055.18,-175.361 1085.72,-176.104"/>
<polygon fill="black" stroke="black" points="1085.72,-179.605 1095.8,-176.349 1085.89,-172.607 1085.72,-179.605"/>
<text text-anchor="middle" x="1031.02" y="-179.976" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g>
<!-- clean wait&#45;&gt;clean failed -->
<g id="edge28" class="edge"><title>clean wait&#45;&gt;clean failed</title>
<path fill="none" stroke="black" d="M797.117,-193.449C825.68,-187.71 867.901,-182.218 904,-189.55 911.895,-191.154 920.046,-193.851 927.674,-196.894"/>
<polygon fill="black" stroke="black" points="926.429,-200.168 937,-200.879 929.179,-193.731 926.429,-200.168"/>
<text text-anchor="middle" x="866" y="-192.95" font-family="Times,serif" font-size="12.00">abort (via API)</text>
<g id="edge32" class="edge"><title>clean wait&#45;&gt;clean failed</title>
<path fill="none" stroke="black" d="M947.568,-162.319C959.219,-159.957 972.075,-157.753 984.019,-156.576 1025.59,-152.476 1036.7,-150.403 1078.02,-156.576 1085.83,-157.742 1093.97,-159.615 1101.81,-161.774"/>
<polygon fill="black" stroke="black" points="1100.95,-165.169 1111.53,-164.63 1102.92,-158.452 1100.95,-165.169"/>
<text text-anchor="middle" x="1031.02" y="-159.976" font-family="Times,serif" font-size="12.00">abort (via API)</text>
</g>
<!-- inspect failed&#45;&gt;manageable -->
<g id="edge35" class="edge"><title>inspect failed&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M716.878,-56.0731C633.005,-61.4772 456.973,-76.3869 406,-108.55 386.09,-121.113 370.544,-142.979 360.419,-160.525"/>
<polygon fill="black" stroke="black" points="357.104,-159.292 355.359,-169.743 363.24,-162.661 357.104,-159.292"/>
<text text-anchor="middle" x="551" y="-81.95" font-family="Times,serif" font-size="12.00">manage (via API)</text>
<g id="edge39" class="edge"><title>inspect failed&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M850.036,-59.343C749.254,-65.0049 543.392,-80.1205 479.359,-111.576 459.714,-121.226 441.689,-138.083 428.7,-152.345"/>
<polygon fill="black" stroke="black" points="425.998,-150.118 422.024,-159.939 431.256,-154.74 425.998,-150.118"/>
<text text-anchor="middle" x="654.359" y="-85.9757" font-family="Times,serif" font-size="12.00">manage (via API)</text>
</g>
<!-- inspect failed&#45;&gt;inspecting -->
<g id="edge36" class="edge"><title>inspect failed&#45;&gt;inspecting</title>
<path fill="none" stroke="black" d="M734.727,-39.2587C723.645,-34.1997 710.52,-29.1276 698,-26.55 665.054,-19.7673 627.033,-19.7478 598.051,-21.3024"/>
<polygon fill="black" stroke="black" points="597.573,-17.8252 587.809,-21.9367 598.006,-24.8118 597.573,-17.8252"/>
<text text-anchor="middle" x="655" y="-29.95" font-family="Times,serif" font-size="12.00">inspect (via API)</text>
<g id="edge40" class="edge"><title>inspect failed&#45;&gt;inspecting</title>
<path fill="none" stroke="black" d="M873.23,-41.8394C860.397,-36.9231 845.45,-32.066 831.359,-29.5757 791.238,-22.4852 745.165,-22.5096 710.224,-24.1608"/>
<polygon fill="black" stroke="black" points="710.019,-20.6667 700.219,-24.6921 710.39,-27.6569 710.019,-20.6667"/>
<text text-anchor="middle" x="778.359" y="-32.9757" font-family="Times,serif" font-size="12.00">inspect (via API)</text>
</g>
<!-- adopt failed&#45;&gt;manageable -->
<g id="edge40" class="edge"><title>adopt failed&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M732.005,-418.332C651.178,-385.257 433.099,-294.955 406,-272.55 386.665,-256.564 370.867,-232.724 360.501,-214.356"/>
<polygon fill="black" stroke="black" points="363.439,-212.432 355.586,-205.32 357.29,-215.777 363.439,-212.432"/>
<text text-anchor="middle" x="551" y="-361.95" font-family="Times,serif" font-size="12.00">manage (via API)</text>
<g id="edge44" class="edge"><title>adopt failed&#45;&gt;manageable</title>
<path fill="none" stroke="black" d="M868.877,-435.057C776.709,-404.07 543.338,-322.609 479.359,-271.576 455.808,-252.79 435.824,-224.525 423.093,-203.895"/>
<polygon fill="black" stroke="black" points="426.078,-202.066 417.93,-195.294 420.076,-205.669 426.078,-202.066"/>
<text text-anchor="middle" x="654.359" y="-378.976" font-family="Times,serif" font-size="12.00">manage (via API)</text>
</g>
<!-- adopt failed&#45;&gt;adopting -->
<g id="edge39" class="edge"><title>adopt failed&#45;&gt;adopting</title>
<path fill="none" stroke="black" d="M728.401,-420.602C718.68,-418.147 707.995,-415.85 698,-414.55 663.074,-410.007 623.054,-409.806 593.757,-410.573"/>
<polygon fill="black" stroke="black" points="593.364,-407.083 583.478,-410.892 593.582,-414.08 593.364,-407.083"/>
<text text-anchor="middle" x="655" y="-417.95" font-family="Times,serif" font-size="12.00">adopt (via API)</text>
<g id="edge43" class="edge"><title>adopt failed&#45;&gt;adopting</title>
<path fill="none" stroke="black" d="M863.905,-436.956C853.333,-434.757 841.99,-432.753 831.359,-431.576 789.48,-426.936 741.654,-426.715 706.438,-427.497"/>
<polygon fill="black" stroke="black" points="706.302,-424 696.396,-427.755 706.482,-430.997 706.302,-424"/>
<text text-anchor="middle" x="778.359" y="-434.976" font-family="Times,serif" font-size="12.00">adopt (via API)</text>
</g>
<!-- rescue wait&#45;&gt;deleting -->
<g id="edge51" class="edge"><title>rescue wait&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M1934.09,-416.616C1941.08,-417.377 1948.22,-418.066 1954.98,-418.576 2054.31,-426.051 2079.76,-416.889 2178.98,-425.576 2473.28,-451.341 2546.99,-463.618 2836.87,-520.576 2841.08,-521.403 2845.45,-522.344 2849.8,-523.338"/>
<polygon fill="black" stroke="black" points="2849.33,-526.822 2859.87,-525.726 2850.94,-520.012 2849.33,-526.822"/>
<text text-anchor="middle" x="2381.98" y="-453.976" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g>
<!-- rescue wait&#45;&gt;rescuing -->
<g id="edge48" class="edge"><title>rescue wait&#45;&gt;rescuing</title>
<path fill="none" stroke="black" d="M1847.79,-399.468C1813.24,-391.462 1761.63,-384.597 1720.16,-400.576 1698.78,-408.813 1679.78,-426.542 1666.54,-441.552"/>
<polygon fill="black" stroke="black" points="1663.58,-439.637 1659.8,-449.534 1668.93,-444.154 1663.58,-439.637"/>
<text text-anchor="middle" x="1769.66" y="-403.976" font-family="Times,serif" font-size="12.00" fill="gray">resume</text>
</g>
<!-- rescue wait&#45;&gt;rescue failed -->
<g id="edge49" class="edge"><title>rescue wait&#45;&gt;rescue failed</title>
<path fill="none" stroke="black" d="M1934.64,-405.04C1966.24,-400.908 2008.85,-394.599 2045.98,-386.576 2054.01,-384.841 2062.46,-382.736 2070.66,-380.538"/>
<polygon fill="black" stroke="black" points="2071.79,-383.858 2080.5,-377.829 2069.93,-377.109 2071.79,-383.858"/>
<text text-anchor="middle" x="2000.48" y="-404.976" font-family="Times,serif" font-size="12.00" fill="gray">fail</text>
</g>
<!-- rescue wait&#45;&gt;rescue failed -->
<g id="edge50" class="edge"><title>rescue wait&#45;&gt;rescue failed</title>
<path fill="none" stroke="black" d="M1912.23,-394.924C1929.41,-383.941 1950.25,-370.973 1954.98,-369.576 1987.92,-359.855 2026.17,-358.338 2057.59,-359.483"/>
<polygon fill="black" stroke="black" points="2057.82,-362.997 2067.97,-359.966 2058.14,-356.005 2057.82,-362.997"/>
<text text-anchor="middle" x="2000.48" y="-372.976" font-family="Times,serif" font-size="12.00">abort (via API)</text>
</g>
<!-- rescue failed&#45;&gt;deleting -->
<g id="edge54" class="edge"><title>rescue failed&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M2166.87,-376.352C2176.75,-378.569 2187.2,-380.774 2196.98,-382.576 2267.59,-395.574 2773.44,-438.939 2836.87,-472.576 2852.86,-481.054 2866.58,-495.982 2876.45,-509.065"/>
<polygon fill="black" stroke="black" points="2873.69,-511.224 2882.36,-517.31 2879.38,-507.144 2873.69,-511.224"/>
<text text-anchor="middle" x="2509.48" y="-427.976" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g>
<!-- rescue failed&#45;&gt;rescuing -->
<g id="edge52" class="edge"><title>rescue failed&#45;&gt;rescuing</title>
<path fill="none" stroke="black" d="M2073.71,-356.084C1995.44,-342.695 1835.22,-326.124 1720.16,-386.576 1696.43,-399.045 1676.5,-422.46 1663.51,-440.749"/>
<polygon fill="black" stroke="black" points="1660.45,-439.014 1657.7,-449.245 1666.23,-442.965 1660.45,-439.014"/>
<text text-anchor="middle" x="1887.07" y="-352.976" font-family="Times,serif" font-size="12.00">rescue (via API)</text>
</g>
<!-- rescue failed&#45;&gt;unrescuing -->
<g id="edge53" class="edge"><title>rescue failed&#45;&gt;unrescuing</title>
<path fill="none" stroke="black" d="M2177.62,-365.576C2220.04,-365.576 2278.54,-365.576 2321.91,-365.576"/>
<polygon fill="black" stroke="black" points="2322.03,-369.076 2332.03,-365.576 2322.03,-362.076 2322.03,-369.076"/>
<text text-anchor="middle" x="2254.48" y="-368.976" font-family="Times,serif" font-size="12.00">unrescue (via API)</text>
</g>
<!-- unrescue failed&#45;&gt;deleting -->
<g id="edge59" class="edge"><title>unrescue failed&#45;&gt;deleting</title>
<path fill="none" stroke="black" d="M2714.36,-397.093C2753.01,-398.627 2801.58,-406.312 2836.87,-431.576 2862.22,-449.724 2877.31,-483.013 2885.35,-506.836"/>
<polygon fill="black" stroke="black" points="2882.02,-507.915 2888.37,-516.395 2888.69,-505.806 2882.02,-507.915"/>
<text text-anchor="middle" x="2784.87" y="-434.976" font-family="Times,serif" font-size="12.00">deleted (via API)</text>
</g>
<!-- unrescue failed&#45;&gt;rescuing -->
<g id="edge57" class="edge"><title>unrescue failed&#45;&gt;rescuing</title>
<path fill="none" stroke="black" d="M2585.58,-396.578C2546.78,-395.801 2496.25,-396.687 2451.98,-403.576 2326.93,-423.035 2304.23,-466.387 2178.98,-484.576 1977.17,-513.883 1922.95,-503.11 1720.16,-481.576 1711,-480.603 1701.3,-479.011 1692.05,-477.218"/>
<polygon fill="black" stroke="black" points="1692.75,-473.789 1682.26,-475.215 1691.35,-480.647 1692.75,-473.789"/>
<text text-anchor="middle" x="2121.48" y="-500.976" font-family="Times,serif" font-size="12.00">rescue (via API)</text>
</g>
<!-- unrescue failed&#45;&gt;unrescuing -->
<g id="edge58" class="edge"><title>unrescue failed&#45;&gt;unrescuing</title>
<path fill="none" stroke="black" d="M2620.75,-383.264C2605.4,-375.316 2585.75,-366.617 2566.98,-362.576 2524.73,-353.479 2475.75,-354.834 2438.95,-358.135"/>
<polygon fill="black" stroke="black" points="2438.53,-354.66 2428.91,-359.111 2439.21,-361.627 2438.53,-354.66"/>
<text text-anchor="middle" x="2509.48" y="-365.976" font-family="Times,serif" font-size="12.00">unrescue (via API)</text>
</g>
</g>
</svg>

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 25 KiB

After

Width:  |  Height:  |  Size: 37 KiB

10
ironic/common/exception.py
View File

@ -762,3 +762,13 @@ class AgentAPIError(IronicException):
class NodeTraitNotFound(IronicException):
_msg_fmt = _("Node %(node_id)s doesn't have a trait '%(trait)s'")
class InstanceRescueFailure(IronicException):
_msg_fmt = _('Failed to rescue instance %(instance)s for node '
'%(node)s: %(reason)s')
class InstanceUnrescueFailure(IronicException):
_msg_fmt = _('Failed to unrescue instance %(instance)s for node '
'%(node)s: %(reason)s')

2
ironic/common/release_mappings.py
View File

@ -122,7 +122,7 @@ RELEASE_MAPPING = {
},
'master': {
'api': '1.36',
'rpc': '1.42',
'rpc': '1.43',
'objects': {
'Node': ['1.22'],
'Conductor': ['1.2'],

69
ironic/common/states.py
View File

@ -48,6 +48,8 @@ VERBS = {
'abort': 'abort',
'clean': 'clean',
'adopt': 'adopt',
'rescue': 'rescue',
'unrescue': 'unrescue',
}
""" Mapping of state-changing events that are PUT to the REST API
@ -208,17 +210,18 @@ UNRESCUING = 'unrescuing'
""" Node is being restored from rescue mode (to active state). """
UPDATE_ALLOWED_STATES = (DEPLOYFAIL, INSPECTING, INSPECTFAIL, CLEANFAIL, ERROR,
VERIFYING, ADOPTFAIL)
VERIFYING, ADOPTFAIL, RESCUEFAIL, UNRESCUEFAIL)
"""Transitional states in which we allow updating a node."""
DELETE_ALLOWED_STATES = (AVAILABLE, MANAGEABLE, ENROLL, ADOPTFAIL)
"""States in which node deletion is allowed."""
STABLE_STATES = (ENROLL, MANAGEABLE, AVAILABLE, ACTIVE, ERROR)
STABLE_STATES = (ENROLL, MANAGEABLE, AVAILABLE, ACTIVE, ERROR, RESCUE)
"""States that will not transition unless receiving a request."""
UNSTABLE_STATES = (DEPLOYING, DEPLOYWAIT, CLEANING, CLEANWAIT, VERIFYING,
DELETING, INSPECTING, ADOPTING)
DELETING, INSPECTING, ADOPTING, RESCUING, RESCUEWAIT,
UNRESCUING)
"""States that can be changed without external request."""
##############
@ -294,6 +297,13 @@ machine.add_state(INSPECTFAIL, target=MANAGEABLE, **watchers)
machine.add_state(ADOPTING, target=ACTIVE, **watchers)
machine.add_state(ADOPTFAIL, target=ACTIVE, **watchers)
# rescue states
machine.add_state(RESCUING, target=RESCUE, **watchers)
machine.add_state(RESCUEWAIT, target=RESCUE, **watchers)
machine.add_state(RESCUEFAIL, target=RESCUE, **watchers)
machine.add_state(UNRESCUING, target=ACTIVE, **watchers)
machine.add_state(UNRESCUEFAIL, target=ACTIVE, **watchers)
# A deployment may fail
machine.add_transition(DEPLOYING, DEPLOYFAIL, 'fail')
@ -389,6 +399,59 @@ machine.add_transition(INSPECTFAIL, MANAGEABLE, 'manage')
# Reinitiate the inspect after inspectfail.
machine.add_transition(INSPECTFAIL, INSPECTING, 'inspect')
# A provisioned node may have a rescue initiated.
machine.add_transition(ACTIVE, RESCUING, 'rescue')
# A rescue may succeed.
machine.add_transition(RESCUING, RESCUE, 'done')
# A rescue may also wait on external callbacks
machine.add_transition(RESCUING, RESCUEWAIT, 'wait')
machine.add_transition(RESCUEWAIT, RESCUING, 'resume')
# A rescued node may be re-rescued.
machine.add_transition(RESCUE, RESCUING, 'rescue')
# A rescued node may be deleted.
machine.add_transition(RESCUE, DELETING, 'delete')
# A rescue may fail.
machine.add_transition(RESCUEWAIT, RESCUEFAIL, 'fail')
machine.add_transition(RESCUING, RESCUEFAIL, 'fail')
# While waiting for a rescue step to be finished, rescuing may be aborted
machine.add_transition(RESCUEWAIT, RESCUEFAIL, 'abort')
# A failed rescue may be re-rescued.
machine.add_transition(RESCUEFAIL, RESCUING, 'rescue')
# A failed rescue may be unrescued.
machine.add_transition(RESCUEFAIL, UNRESCUING, 'unrescue')
# A failed rescue may be deleted.
machine.add_transition(RESCUEFAIL, DELETING, 'delete')
# A rescuewait node may be deleted.
machine.add_transition(RESCUEWAIT, DELETING, 'delete')
# A rescued node may be unrescued.
machine.add_transition(RESCUE, UNRESCUING, 'unrescue')
# An unrescuing node may succeed
machine.add_transition(UNRESCUING, ACTIVE, 'done')
# An unrescuing node may fail
machine.add_transition(UNRESCUING, UNRESCUEFAIL, 'fail')
# A failed unrescue may be re-rescued
machine.add_transition(UNRESCUEFAIL, RESCUING, 'rescue')
# A failed unrescue may be re-unrescued
machine.add_transition(UNRESCUEFAIL, UNRESCUING, 'unrescue')
# A failed unrescue may be deleted.
machine.add_transition(UNRESCUEFAIL, DELETING, 'delete')
# Start power credentials verification
machine.add_transition(ENROLL, VERIFYING, 'manage')

252
ironic/conductor/manager.py
View File

@ -94,7 +94,7 @@ class ConductorManager(base_manager.BaseConductorManager):
# NOTE(rloo): This must be in sync with rpcapi.ConductorAPI's.
# NOTE(pas-ha): This also must be in sync with
# ironic.common.release_mappings.RELEASE_MAPPING['master']
RPC_API_VERSION = '1.42'
RPC_API_VERSION = '1.43'
target = messaging.Target(version=RPC_API_VERSION)
@ -527,6 +527,228 @@ class ConductorManager(base_manager.BaseConductorManager):
return get_vendor_passthru_metadata(vendor.driver_routes)
@METRICS.timer('ConductorManager.do_node_rescue')
@messaging.expected_exceptions(exception.NoFreeConductorWorker,
exception.NodeInMaintenance,
exception.NodeLocked,
exception.InstanceRescueFailure,
exception.InvalidStateRequested,
exception.UnsupportedDriverExtension
)
def do_node_rescue(self, context, node_id, rescue_password):
"""RPC method to rescue an existing node deployment.
Validate driver specific information synchronously, and then
spawn a background worker to rescue the node asynchronously.
:param context: an admin context.
:param node_id: the id or uuid of a node.
:param rescue_password: string to be set as the password inside the
rescue environment.
:raises: InstanceRescueFailure if the node cannot be placed into
rescue mode.
:raises: InvalidStateRequested if the state transition is not supported
or allowed.
:raises: NoFreeConductorWorker when there is no free worker to start
async task.
:raises: NodeLocked if the node is locked by another conductor.
:raises: NodeInMaintenance if the node is in maintenance mode.
:raises: UnsupportedDriverExtension if rescue interface is not
supported by the driver.
"""
LOG.debug("RPC do_node_rescue called for node %s.", node_id)
with task_manager.acquire(context,
node_id, purpose='node rescue') as task:
node = task.node
if node.maintenance:
raise exception.NodeInMaintenance(op=_('rescuing'),
node=node.uuid)
if not getattr(task.driver, 'rescue', None):
raise exception.UnsupportedDriverExtension(
driver=node.driver, extension='rescue')
# driver validation may check rescue_password, so save it on the
# node early
instance_info = node.instance_info
instance_info['rescue_password'] = rescue_password
node.instance_info = instance_info
node.save()
try:
task.driver.power.validate(task)
task.driver.rescue.validate(task)
task.driver.network.validate(task)
except (exception.InvalidParameterValue,
exception.UnsupportedDriverExtension,
exception.MissingParameterValue) as e:
utils.remove_node_rescue_password(node, save=True)
raise exception.InstanceRescueFailure(
instance=node.instance_uuid,
node=node.uuid,
reason=_("Validation failed. Error: %s") % e)
try:
task.process_event(
'rescue',
callback=self._spawn_worker,
call_args=(self._do_node_rescue, task),
err_handler=utils.spawn_rescue_error_handler)
except exception.InvalidState:
utils.remove_node_rescue_password(node, save=True)
raise exception.InvalidStateRequested(
action='rescue', node=node.uuid,
state=node.provision_state)
def _do_node_rescue(self, task):
"""Internal RPC method to rescue an existing node deployment."""
node = task.node
def handle_failure(e, errmsg, log_func=LOG.error):
utils.remove_node_rescue_password(node, save=False)
node.last_error = errmsg % e
task.process_event('fail')
log_func('Error while performing rescue operation for node '
'%(node)s with instance %(instance)s: %(err)s',
{'node': node.uuid, 'instance': node.instance_uuid,
'err': e})
try:
next_state = task.driver.rescue.rescue(task)
except exception.IronicException as e:
with excutils.save_and_reraise_exception():
handle_failure(e,
_('Failed to rescue: %s'))
except Exception as e:
with excutils.save_and_reraise_exception():
handle_failure(e,
_('Failed to rescue. Exception: %s'),
log_func=LOG.exception)
if next_state == states.RESCUEWAIT:
task.process_event('wait')
elif next_state == states.RESCUE:
task.process_event('done')
else:
error = (_("Driver returned unexpected state %s") % next_state)
handle_failure(error,
_('Failed to rescue: %s'))
@METRICS.timer('ConductorManager.do_node_unrescue')
@messaging.expected_exceptions(exception.NoFreeConductorWorker,
exception.NodeInMaintenance,
exception.NodeLocked,
exception.InstanceUnrescueFailure,
exception.InvalidStateRequested,
exception.UnsupportedDriverExtension
)
def do_node_unrescue(self, context, node_id):
"""RPC method to unrescue a node in rescue mode.
Validate driver specific information synchronously, and then
spawn a background worker to unrescue the node asynchronously.
:param context: an admin context.
:param node_id: the id or uuid of a node.
:raises: InstanceUnrescueFailure if the node fails to be unrescued
:raises: InvalidStateRequested if the state transition is not supported
or allowed.
:raises: NoFreeConductorWorker when there is no free worker to start
async task
:raises: NodeLocked if the node is locked by another conductor.
:raises: NodeInMaintenance if the node is in maintenance mode.
:raises: UnsupportedDriverExtension if rescue interface is not
supported by the driver.
"""
LOG.debug("RPC do_node_unrescue called for node %s.", node_id)
with task_manager.acquire(context, node_id,
purpose='node unrescue') as task:
node = task.node
if node.maintenance:
raise exception.NodeInMaintenance(op=_('unrescuing'),
node=node.uuid)
if not getattr(task.driver, 'rescue', None):
raise exception.UnsupportedDriverExtension(
driver=node.driver, extension='rescue')
try:
task.driver.power.validate(task)
except (exception.InvalidParameterValue,
exception.MissingParameterValue) as e:
raise exception.InstanceUnrescueFailure(
instance=node.instance_uuid,
node=node.uuid,
reason=_("Validation failed. Error: %s") % e)
try:
task.process_event(
'unrescue',
callback=self._spawn_worker,
call_args=(self._do_node_unrescue, task),
err_handler=utils.provisioning_error_handler)
except exception.InvalidState:
raise exception.InvalidStateRequested(
action='unrescue', node=node.uuid,
state=node.provision_state)
def _do_node_unrescue(self, task):
"""Internal RPC method to unrescue a node in rescue mode."""
node = task.node
def handle_failure(e, errmsg, log_func=LOG.error):
node.last_error = errmsg % e
task.process_event('fail')
log_func('Error while performing unrescue operation for node '
'%(node)s with instance %(instance)s: %(err)s',
{'node': node.uuid, 'instance': node.instance_uuid,
'err': e})
try:
next_state = task.driver.rescue.unrescue(task)
except exception.IronicException as e:
with excutils.save_and_reraise_exception():
handle_failure(e,
_('Failed to unrescue: %s'))
except Exception as e:
with excutils.save_and_reraise_exception():
handle_failure(e,
_('Failed to unrescue. Exception: %s'),
log_func=LOG.exception)
if next_state == states.ACTIVE:
task.process_event('done')
else:
error = (_("Driver returned unexpected state %s") % next_state)
handle_failure(error,
_('Failed to unrescue: %s'))
@task_manager.require_exclusive_lock
def _do_node_rescue_abort(self, task):
"""Internal method to abort an ongoing rescue operation.
:param task: a TaskManager instance with an exclusive lock
"""
node = task.node
try:
task.driver.rescue.clean_up(task)
except Exception as e:
LOG.exception('Failed to clean up rescue for node %(node)s '
'after aborting the operation. Error: %(err)s',
{'node': node.uuid, 'err': e})
error_msg = _('Failed to clean up rescue after aborting '
'the operation')
node.refresh()
node.last_error = error_msg
node.maintenance = True
node.maintenance_reason = error_msg
node.save()
return
info_message = _('Rescue operation aborted for node %s.') % node.uuid
last_error = _('By request, the rescue operation was aborted.')
node.refresh()
node.last_error = last_error
node.save()
LOG.info(info_message)
@METRICS.timer('ConductorManager.do_node_deploy')
@messaging.expected_exceptions(exception.NoFreeConductorWorker,
exception.NodeLocked,
@ -657,7 +879,8 @@ class ConductorManager(base_manager.BaseConductorManager):
task.process_event(
'delete',
callback=self._spawn_worker,
call_args=(self._do_node_tear_down, task),
call_args=(self._do_node_tear_down, task,
task.node.provision_state),
err_handler=utils.provisioning_error_handler)
except exception.InvalidState:
raise exception.InvalidStateRequested(
@ -665,10 +888,21 @@ class ConductorManager(base_manager.BaseConductorManager):
state=task.node.provision_state)
@task_manager.require_exclusive_lock
def _do_node_tear_down(self, task):
"""Internal RPC method to tear down an existing node deployment."""
def _do_node_tear_down(self, task, initial_state):
"""Internal RPC method to tear down an existing node deployment.
:param task: a task from TaskManager.
:param initial_state: The initial provision state from which node
has moved into deleting state.
"""
node = task.node
try:
if (initial_state in (states.RESCUEWAIT, states.RESCUE,
states.UNRESCUEFAIL, states.RESCUEFAIL)):
# Perform rescue clean up. Rescue clean up will remove
# rescuing network as well.
task.driver.rescue.clean_up(task)
task.driver.deploy.clean_up(task)
task.driver.deploy.tear_down(task)
except Exception as e:
@ -1216,6 +1450,16 @@ class ConductorManager(base_manager.BaseConductorManager):
target_state=target_state)
return
if (action == states.VERBS['abort'] and
node.provision_state == states.RESCUEWAIT):
utils.remove_node_rescue_password(node, save=True)
task.process_event(
'abort',
callback=self._spawn_worker,
call_args=(self._do_node_rescue_abort, task),
err_handler=utils.provisioning_error_handler)
return
try:
task.process_event(action)
except exception.InvalidState:

44
ironic/conductor/rpcapi.py
View File

@ -91,13 +91,14 @@ class ConductorAPI(object):
| 1.40 - Added inject_nmi
| 1.41 - Added create_port
| 1.42 - Added optional agent_version to heartbeat
| 1.43 - Added do_node_rescue, do_node_unrescue and can_send_rescue
"""
# NOTE(rloo): This must be in sync with manager.ConductorManager's.
# NOTE(pas-ha): This also must be in sync with
# ironic.common.release_mappings.RELEASE_MAPPING['master']
RPC_API_VERSION = '1.42'
RPC_API_VERSION = '1.43'
def __init__(self, topic=None):
super(ConductorAPI, self).__init__()
@ -158,6 +159,10 @@ class ConductorAPI(object):
"""Return whether the RPCAPI supports the create_port method."""
return self.client.can_send_version("1.41")
def can_send_rescue(self):
"""Return whether the RPCAPI supports node rescue methods."""
return self.client.can_send_version("1.43")
def create_node(self, context, node_obj, topic=None):
"""Synchronously, have a conductor validate and create a node.
@ -975,3 +980,40 @@ class ConductorAPI(object):
"""
cctxt = self.client.prepare(topic=topic or self.topic, version='1.38')
return cctxt.call(context, 'vif_list', node_id=node_id)
def do_node_rescue(self, context, node_id, rescue_password, topic=None):
"""Signal to conductor service to perform a rescue.
:param context: request context.
:param node_id: node ID or UUID.
:param rescue_password: A string representing the password to be set
inside the rescue environment.
:param topic: RPC topic. Defaults to self.topic.
:raises: InstanceRescueFailure
:raises: NoFreeConductorWorker when there is no free worker to start
async task.
The node must already be configured and in the appropriate
state before this method is called.
"""
cctxt = self.client.prepare(topic=topic or self.topic, version='1.43')
return cctxt.call(context, 'do_node_rescue', node_id=node_id,
rescue_password=rescue_password)
def do_node_unrescue(self, context, node_id, topic=None):
"""Signal to conductor service to perform an unrescue.
:param context: request context.
:param node_id: node ID or UUID.
:param topic: RPC topic. Defaults to self.topic.
:raises: InstanceUnrescueFailure
:raises: NoFreeConductorWorker when there is no free worker to start
async task.
The node must already be configured and in the appropriate
state before this method is called.
"""
cctxt = self.client.prepare(topic=topic or self.topic, version='1.43')
return cctxt.call(context, 'do_node_unrescue', node_id=node_id)

38
ironic/conductor/utils.py
View File

@ -398,13 +398,26 @@ def cleaning_error_handler(task, msg, tear_down_cleaning=True,
task.process_event('fail', target_state=target_state)
def spawn_cleaning_error_handler(e, node):
"""Handle spawning error for node cleaning."""
def _spawn_error_handler(e, node, state):
"""Handle spawning error for node."""
if isinstance(e, exception.NoFreeConductorWorker):
node.last_error = (_("No free conductor workers available"))
node.save()
LOG.warning("No free conductor workers available to perform "
"cleaning on node %(node)s", {'node': node.uuid})
"%(state)s on node %(node)s",
{'state': state, 'node': node.uuid})
def spawn_cleaning_error_handler(e, node):
"""Handle spawning error for node cleaning."""
_spawn_error_handler(e, node, states.CLEANING)
def spawn_rescue_error_handler(e, node):
"""Handle spawning error for node rescue."""
if isinstance(e, exception.NoFreeConductorWorker):
remove_node_rescue_password(node, save=False)
_spawn_error_handler(e, node, states.RESCUE)
def power_state_error_handler(e, node, power_state):
@ -652,3 +665,22 @@ def validate_port_physnet(task, port_obj):
raise exception.Conflict(
msg % {'portgroup': portgroup.uuid, 'physnet': port_physnet,
'pg_physnet': pg_physnet})
def remove_node_rescue_password(node, save=True):
"""Helper to remove rescue password from a node.
Removes rescue password from node. It saves node by default.
If node should not be saved, then caller needs to explicitly
indicate it.
:param node: an Ironic node object.
:param save: Boolean; True (default) to save the node; False
otherwise.
"""
instance_info = node.instance_info
if 'rescue_password' in instance_info:
del instance_info['rescue_password']
node.instance_info = instance_info
if save:
node.save()

2
ironic/tests/unit/conductor/mgr_utils.py
View File

@ -86,7 +86,7 @@ class CommonMixIn(object):
if node is None:
node = self._create_node(**node_attrs)
task = mock.Mock(spec_set=['node', 'release_resources',
'spawn_after', 'process_event'])
'spawn_after', 'process_event', 'driver'])
task.node = node
return task

353
ironic/tests/unit/conductor/test_manager.py
View File

@ -580,6 +580,7 @@ class UpdateNodeTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):
'network_interface': UpdateInterfaces('flat', 'noop'),
'power_interface': UpdateInterfaces(None, 'fake'),
'raid_interface': UpdateInterfaces(None, 'fake'),
'rescue_interface': UpdateInterfaces(None, 'no-rescue'),
'storage_interface': UpdateInterfaces('noop', 'cinder'),
}
@ -1647,7 +1648,8 @@ class DoNodeDeployTearDownTestCase(mgr_utils.ServiceSetUpMixin,
self._start_service()
mock_tear_down.side_effect = exception.InstanceDeployFailure('test')
self.assertRaises(exception.InstanceDeployFailure,
self.service._do_node_tear_down, task)
self.service._do_node_tear_down, task,
node.provision_state)
node.refresh()
self.assertEqual(states.ERROR, node.provision_state)
self.assertEqual(states.NOSTATE, node.target_provision_state)
@ -1670,7 +1672,7 @@ class DoNodeDeployTearDownTestCase(mgr_utils.ServiceSetUpMixin,
task = task_manager.TaskManager(self.context, node.uuid)
self._start_service()
self.service._do_node_tear_down(task)
self.service._do_node_tear_down(task, node.provision_state)
node.refresh()
# Node will be moved to AVAILABLE after cleaning, not tested here
self.assertEqual(states.CLEANING, node.provision_state)
@ -1682,12 +1684,15 @@ class DoNodeDeployTearDownTestCase(mgr_utils.ServiceSetUpMixin,
mock_tear_down.assert_called_once_with(mock.ANY)
mock_clean.assert_called_once_with(mock.ANY)
@mock.patch('ironic.drivers.modules.fake.FakeRescue.clean_up')
@mock.patch('ironic.conductor.manager.ConductorManager._do_node_clean')
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.tear_down')
def _test_do_node_tear_down_from_state(self, init_state, mock_tear_down,
mock_clean):
def _test_do_node_tear_down_from_state(self, init_state, is_rescue_state,
mock_tear_down, mock_clean,
mock_rescue_clean):
node = obj_utils.create_test_node(
self.context, driver='fake', uuid=uuidutils.generate_uuid(),
self.context, driver='fake-hardware',
uuid=uuidutils.generate_uuid(),
provision_state=init_state,
target_provision_state=states.AVAILABLE,
driver_internal_info={'is_whole_disk_image': False})
@ -1703,12 +1708,21 @@ class DoNodeDeployTearDownTestCase(mgr_utils.ServiceSetUpMixin,
self.assertEqual({}, node.instance_info)
mock_tear_down.assert_called_once_with(mock.ANY)
mock_clean.assert_called_once_with(mock.ANY)
if is_rescue_state:
mock_rescue_clean.assert_called_once_with(mock.ANY)
else:
self.assertFalse(mock_rescue_clean.called)
def test__do_node_tear_down_from_valid_states(self):
valid_states = [states.ACTIVE, states.DEPLOYWAIT, states.DEPLOYFAIL,
states.ERROR]
for state in valid_states:
self._test_do_node_tear_down_from_state(state)
self._test_do_node_tear_down_from_state(state, False)
valid_rescue_states = [states.RESCUEWAIT, states.RESCUE,
states.UNRESCUEFAIL, states.RESCUEFAIL]
for state in valid_rescue_states:
self._test_do_node_tear_down_from_state(state, True)
# NOTE(deva): partial tear-down was broken. A node left in a state of
# DELETING could not have tear_down called on it a second time
@ -2746,6 +2760,333 @@ class DoNodeCleanTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):
self.assertEqual(0, step_index)
class DoNodeRescueTestCase(mgr_utils.CommonMixIn, mgr_utils.ServiceSetUpMixin,
db_base.DbTestCase):
@mock.patch('ironic.conductor.task_manager.acquire', autospec=True)
def test_do_node_rescue(self, mock_acquire):
self._start_service()
task = self._create_task(
node_attrs=dict(driver='fake-hardware',
provision_state=states.ACTIVE,
instance_info={}))
mock_acquire.side_effect = self._get_acquire_side_effect(task)
self.service.do_node_rescue(self.context, task.node.uuid,
"password")
task.process_event.assert_called_once_with(
'rescue',
callback=self.service._spawn_worker,
call_args=(self.service._do_node_rescue, task),
err_handler=conductor_utils.spawn_rescue_error_handler)
self.assertIn('rescue_password', task.node.instance_info)
def test_do_node_rescue_invalid_state(self):
self._start_service()
node = obj_utils.create_test_node(self.context, driver='fake-hardware',
network_interface='noop',
provision_state=states.AVAILABLE,
instance_info={})
exc = self.assertRaises(messaging.rpc.ExpectedException,
self.service.do_node_rescue,
self.context, node.uuid, "password")
node.refresh()
self.assertNotIn('rescue_password', node.instance_info)
self.assertEqual(exception.InvalidStateRequested, exc.exc_info[0])
def _test_do_node_rescue_when_validate_fail(self, mock_validate):
# InvalidParameterValue should be re-raised as InstanceRescueFailure
mock_validate.side_effect = exception.InvalidParameterValue('error')
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.ACTIVE,
target_provision_state=states.NOSTATE,
instance_info={})
exc = self.assertRaises(messaging.rpc.ExpectedException,
self.service.do_node_rescue,
self.context, node.uuid, "password")
node.refresh()
self.assertNotIn('rescue_password', node.instance_info)
# Compare true exception hidden by @messaging.expected_exceptions
self.assertEqual(exception.InstanceRescueFailure, exc.exc_info[0])
@mock.patch('ironic.drivers.modules.fake.FakeRescue.validate')
def test_do_node_rescue_when_rescue_validate_fail(self, mock_validate):
self._test_do_node_rescue_when_validate_fail(mock_validate)
@mock.patch('ironic.drivers.modules.fake.FakePower.validate')
def test_do_node_rescue_when_power_validate_fail(self, mock_validate):
self._test_do_node_rescue_when_validate_fail(mock_validate)
@mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate')
def test_do_node_rescue_when_network_validate_fail(self, mock_validate):
self._test_do_node_rescue_when_validate_fail(mock_validate)
def test_do_node_rescue_not_supported(self):
node = obj_utils.create_test_node(
self.context, driver='fake',
provision_state=states.ACTIVE,
target_provision_state=states.NOSTATE,
instance_info={})
exc = self.assertRaises(messaging.rpc.ExpectedException,
self.service.do_node_rescue,
self.context, node.uuid, "password")
self.assertEqual(exception.UnsupportedDriverExtension,
exc.exc_info[0])
def test_do_node_rescue_maintenance(self):
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.ACTIVE,
maintenance=True,
target_provision_state=states.NOSTATE,
instance_info={})
exc = self.assertRaises(messaging.rpc.ExpectedException,
self.service.do_node_rescue,
self.context, node['uuid'], "password")
# Compare true exception hidden by @messaging.expected_exceptions
self.assertEqual(exception.NodeInMaintenance, exc.exc_info[0])
# This is a sync operation last_error should be None.
self.assertIsNone(node.last_error)
@mock.patch('ironic.drivers.modules.fake.FakeRescue.rescue')
def test__do_node_rescue_returns_rescuewait(self, mock_rescue):
node = obj_utils.create_test_node(self.context, driver='fake-hardware',
provision_state=states.RESCUING,
instance_info={'rescue_password':
'password'})
self._start_service()
with task_manager.TaskManager(self.context, node.uuid) as task:
mock_rescue.return_value = states.RESCUEWAIT
self.service._do_node_rescue(task)
node.refresh()
self.assertEqual(states.RESCUEWAIT, node.provision_state)
self.assertEqual(states.RESCUE, node.target_provision_state)
self.assertIn('rescue_password', node.instance_info)
@mock.patch('ironic.drivers.modules.fake.FakeRescue.rescue')
def test__do_node_rescue_returns_rescue(self, mock_rescue):
node = obj_utils.create_test_node(self.context, driver='fake-hardware',
provision_state=states.RESCUING,
instance_info={'rescue_password':
'password'})
self._start_service()
with task_manager.TaskManager(self.context, node.uuid) as task:
mock_rescue.return_value = states.RESCUE
self.service._do_node_rescue(task)
node.refresh()
self.assertEqual(states.RESCUE, node.provision_state)
self.assertEqual(states.NOSTATE, node.target_provision_state)
self.assertIn('rescue_password', node.instance_info)
@mock.patch.object(manager, 'LOG')
@mock.patch('ironic.drivers.modules.fake.FakeRescue.rescue')
def test__do_node_rescue_errors(self, mock_rescue, mock_log):
node = obj_utils.create_test_node(self.context, driver='fake-hardware',
provision_state=states.RESCUING,
instance_info={'rescue_password':
'password'})
self._start_service()
mock_rescue.side_effect = exception.InstanceRescueFailure(
'failed to rescue')
with task_manager.TaskManager(self.context, node.uuid) as task:
self.assertRaises(exception.InstanceRescueFailure,
self.service._do_node_rescue, task)
node.refresh()
self.assertEqual(states.RESCUEFAIL, node.provision_state)
self.assertEqual(states.RESCUE, node.target_provision_state)
self.assertNotIn('rescue_password', node.instance_info)
self.assertTrue(node.last_error.startswith('Failed to rescue'))
self.assertTrue(mock_log.error.called)
@mock.patch.object(manager, 'LOG')
@mock.patch('ironic.drivers.modules.fake.FakeRescue.rescue')
def test__do_node_rescue_bad_state(self, mock_rescue, mock_log):
node = obj_utils.create_test_node(self.context, driver='fake-hardware',
provision_state=states.RESCUING,
instance_info={'rescue_password':
'password'})
self._start_service()
mock_rescue.return_value = states.ACTIVE
with task_manager.TaskManager(self.context, node.uuid) as task:
self.service._do_node_rescue(task)
node.refresh()
self.assertEqual(states.RESCUEFAIL, node.provision_state)
self.assertEqual(states.RESCUE, node.target_provision_state)
self.assertNotIn('rescue_password', node.instance_info)
self.assertTrue(node.last_error.startswith('Failed to rescue'))
self.assertTrue(mock_log.error.called)
@mock.patch('ironic.conductor.task_manager.acquire', autospec=True)
def test_do_node_unrescue(self, mock_acquire):
self._start_service()
task = self._create_task(
node_attrs=dict(driver='fake-hardware',
provision_state=states.RESCUE))
mock_acquire.side_effect = self._get_acquire_side_effect(task)
self.service.do_node_unrescue(self.context, task.node.uuid)
task.process_event.assert_called_once_with(
'unrescue',
callback=self.service._spawn_worker,
call_args=(self.service._do_node_unrescue, task),
err_handler=conductor_utils.provisioning_error_handler)
def test_do_node_unrescue_invalid_state(self):
self._start_service()
node = obj_utils.create_test_node(self.context, driver='fake-hardware',
provision_state=states.AVAILABLE)
exc = self.assertRaises(messaging.rpc.ExpectedException,
self.service.do_node_unrescue,
self.context, node.uuid)
self.assertEqual(exception.InvalidStateRequested, exc.exc_info[0])
@mock.patch('ironic.drivers.modules.fake.FakePower.validate')
def test_do_node_unrescue_validate_fail(self, mock_validate):
# InvalidParameterValue should be re-raised as InstanceUnrescueFailure
mock_validate.side_effect = exception.InvalidParameterValue('error')
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.RESCUE,
target_provision_state=states.NOSTATE)
exc = self.assertRaises(messaging.rpc.ExpectedException,
self.service.do_node_unrescue,
self.context, node.uuid)
# Compare true exception hidden by @messaging.expected_exceptions
self.assertEqual(exception.InstanceUnrescueFailure, exc.exc_info[0])
def test_do_node_unrescue_not_supported(self):
node = obj_utils.create_test_node(
self.context, driver='fake',
provision_state=states.RESCUE,
instance_info={})
exc = self.assertRaises(messaging.rpc.ExpectedException,
self.service.do_node_unrescue,
self.context, node.uuid)
self.assertEqual(exception.UnsupportedDriverExtension,
exc.exc_info[0])
def test_do_node_unrescue_maintenance(self):
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.RESCUE,
maintenance=True,
target_provision_state=states.NOSTATE,
instance_info={})
exc = self.assertRaises(messaging.rpc.ExpectedException,
self.service.do_node_unrescue,
self.context, node.uuid)
# Compare true exception hidden by @messaging.expected_exceptions
self.assertEqual(exception.NodeInMaintenance, exc.exc_info[0])
# This is a sync operation last_error should be None.
node.refresh()
self.assertIsNone(node.last_error)
@mock.patch('ironic.drivers.modules.fake.FakeRescue.unrescue')
def test__do_node_unrescue(self, mock_unrescue):
node = obj_utils.create_test_node(self.context, driver='fake-hardware',
provision_state=states.UNRESCUING,
target_provision_state=states.ACTIVE,
instance_info={})
self._start_service()
with task_manager.TaskManager(self.context, node.uuid) as task:
mock_unrescue.return_value = states.ACTIVE
self.service._do_node_unrescue(task)
node.refresh()
self.assertEqual(states.ACTIVE, node.provision_state)
self.assertEqual(states.NOSTATE, node.target_provision_state)
@mock.patch.object(manager, 'LOG')
@mock.patch('ironic.drivers.modules.fake.FakeRescue.unrescue')
def test__do_node_unrescue_ironic_error(self, mock_unrescue, mock_log):
node = obj_utils.create_test_node(self.context, driver='fake-hardware',
provision_state=states.UNRESCUING,
target_provision_state=states.ACTIVE,
instance_info={})
self._start_service()
mock_unrescue.side_effect = exception.InstanceUnrescueFailure(
'Unable to unrescue')
with task_manager.TaskManager(self.context, node.uuid) as task:
self.assertRaises(exception.InstanceUnrescueFailure,
self.service._do_node_unrescue, task)
node.refresh()
self.assertEqual(states.UNRESCUEFAIL, node.provision_state)
self.assertEqual(states.ACTIVE, node.target_provision_state)
self.assertTrue('Unable to unrescue' in node.last_error)
self.assertTrue(mock_log.error.called)
@mock.patch.object(manager, 'LOG')
@mock.patch('ironic.drivers.modules.fake.FakeRescue.unrescue')
def test__do_node_unrescue_other_error(self, mock_unrescue, mock_log):
node = obj_utils.create_test_node(self.context, driver='fake-hardware',
provision_state=states.UNRESCUING,
target_provision_state=states.ACTIVE,
instance_info={})
self._start_service()
mock_unrescue.side_effect = RuntimeError('Some failure')
with task_manager.TaskManager(self.context, node.uuid) as task:
self.assertRaises(RuntimeError,
self.service._do_node_unrescue, task)
node.refresh()
self.assertEqual(states.UNRESCUEFAIL, node.provision_state)
self.assertEqual(states.ACTIVE, node.target_provision_state)
self.assertTrue('Some failure' in node.last_error)
self.assertTrue(mock_log.exception.called)
@mock.patch('ironic.drivers.modules.fake.FakeRescue.unrescue')
def test__do_node_unrescue_bad_state(self, mock_unrescue):
node = obj_utils.create_test_node(self.context, driver='fake-hardware',
provision_state=states.UNRESCUING,
instance_info={})
self._start_service()
mock_unrescue.return_value = states.RESCUEWAIT
with task_manager.TaskManager(self.context, node.uuid) as task:
self.service._do_node_unrescue(task)
node.refresh()
self.assertEqual(states.UNRESCUEFAIL, node.provision_state)
self.assertEqual(states.ACTIVE, node.target_provision_state)
self.assertTrue('Driver returned unexpected state' in
node.last_error)
@mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker')
def test_provision_rescue_abort(self, mock_spawn):
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.RESCUEWAIT,
target_provision_state=states.RESCUE,
instance_info={'rescue_password': 'password'})
self._start_service()
self.service.do_provisioning_action(self.context, node.uuid, 'abort')
node.refresh()
self.assertEqual(states.RESCUEFAIL, node.provision_state)
self.assertIsNone(node.last_error)
self.assertNotIn('rescue_password', node.instance_info)
mock_spawn.assert_called_with(self.service._do_node_rescue_abort,
mock.ANY)
@mock.patch.object(fake.FakeRescue, 'clean_up', autospec=True)
def test__do_node_rescue_abort(self, clean_up_mock):
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.RESCUEFAIL,
target_provision_state=states.RESCUE)
with task_manager.acquire(self.context, node.uuid) as task:
self.service._do_node_rescue_abort(task)
clean_up_mock.assert_called_once_with(task.driver.rescue, task)
self.assertIsNotNone(task.node.last_error)
self.assertFalse(task.node.maintenance)
@mock.patch.object(fake.FakeRescue, 'clean_up', autospec=True)
def test__do_node_rescue_abort_clean_up_fail(self, clean_up_mock):
clean_up_mock.side_effect = Exception('Surprise')
node = obj_utils.create_test_node(
self.context, driver='fake-hardware',
provision_state=states.RESCUEFAIL)
with task_manager.acquire(self.context, node.uuid) as task:
self.service._do_node_rescue_abort(task)
clean_up_mock.assert_called_once_with(task.driver.rescue, task)
self.assertIsNotNone(task.node.last_error)
self.assertIsNotNone(task.node.maintenance_reason)
self.assertTrue(task.node.maintenance)
@mgr_utils.mock_record_keepalive
class DoNodeVerifyTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):
@mock.patch('ironic.objects.node.NodeCorrectedPowerStateNotification')

28
ironic/tests/unit/conductor/test_rpcapi.py
View File

@ -499,3 +499,31 @@ class RPCAPITestCase(db_base.DbTestCase):
'call',
node_id='fake-node',
version='1.38')
def test_do_node_rescue(self):
self._test_rpcapi('do_node_rescue',
'call',
version='1.43',
node_id=self.fake_node['uuid'],
rescue_password="password")
def test_do_node_unrescue(self):
self._test_rpcapi('do_node_unrescue',
'call',
version='1.43',
node_id=self.fake_node['uuid'])
def _test_can_send_rescue(self, can_send):
rpcapi = conductor_rpcapi.ConductorAPI(topic='fake-topic')
with mock.patch.object(rpcapi.client,
"can_send_version") as mock_can_send_version:
mock_can_send_version.return_value = can_send
result = rpcapi.can_send_rescue()
self.assertEqual(can_send, result)
mock_can_send_version.assert_called_once_with("1.43")
def test_can_send_rescue_true(self):
self._test_can_send_rescue(True)
def test_can_send_rescue_false(self):
self._test_can_send_rescue(False)

43
ironic/tests/unit/conductor/test_utils.py
View File

@ -1201,6 +1201,25 @@ class ErrorHandlersTestCase(tests_base.TestCase):
self.assertFalse(self.node.save.called)
self.assertFalse(log_mock.warning.called)
@mock.patch.object(conductor_utils, 'LOG')
def test_spawn_rescue_error_handler_no_worker(self, log_mock):
exc = exception.NoFreeConductorWorker()
self.node.instance_info = {'rescue_password': 'pass'}
conductor_utils.spawn_rescue_error_handler(exc, self.node)
self.node.save.assert_called_once_with()
self.assertIn('No free conductor workers', self.node.last_error)
self.assertTrue(log_mock.warning.called)
self.assertNotIn('rescue_password', self.node.instance_info)
@mock.patch.object(conductor_utils, 'LOG')
def test_spawn_rescue_error_handler_other_error(self, log_mock):
exc = Exception('foo')
self.node.instance_info = {'rescue_password': 'pass'}
conductor_utils.spawn_rescue_error_handler(exc, self.node)
self.assertFalse(self.node.save.called)
self.assertFalse(log_mock.warning.called)
self.assertIn('rescue_password', self.node.instance_info)
@mock.patch.object(conductor_utils, 'LOG')
def test_power_state_error_handler_no_worker(self, log_mock):
exc = exception.NoFreeConductorWorker()
@ -1535,3 +1554,27 @@ class ValidatePortPhysnetTestCase(db_base.DbTestCase):
current_physnet='physnet1',
new_physnet=None,
valid=False)
class MiscTestCase(db_base.DbTestCase):
def setUp(self):
super(MiscTestCase, self).setUp()
self.node = obj_utils.create_test_node(
self.context,
driver='fake',
instance_info={'rescue_password': 'pass'})
def _test_remove_node_rescue_password(self, save=True):
conductor_utils.remove_node_rescue_password(self.node, save=save)
self.assertNotIn('rescue_password', self.node.instance_info)
self.node.refresh()
if save:
self.assertNotIn('rescue_password', self.node.instance_info)
else:
self.assertIn('rescue_password', self.node.instance_info)
def test_remove_node_rescue_password_save_true(self):
self._test_remove_node_rescue_password(save=True)
def test_remove_node_rescue_password_save_false(self):
self._test_remove_node_rescue_password(save=False)