From 985c7bf38b8448e342c9547b5712fe7272296fd0 Mon Sep 17 00:00:00 2001 From: gholt Date: Tue, 12 Nov 2013 18:00:15 +0000 Subject: [PATCH] Fix probe test Fix for a probe test that failed every once in a while due to the early-majority change previously committed. Sometimes a write would return success before the third node had succeeded and the probe test would look for on-disk evidence and fail, when it would've been fine had it waited just a bit longer for the third node to complete. Since there's no real way for the probe test to know when all three nodes are done, I just made it retry once a second for several seconds before reporting an error. There may be more tests like this we'll have to fix as we run across them. Change-Id: I749e43d4580a7c726a9a8648f71bafefa70a05f5 --- test/probe/test_object_failures.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/probe/test_object_failures.py b/test/probe/test_object_failures.py index 4ad8a28af1..c6dbccf5ae 100755 --- a/test/probe/test_object_failures.py +++ b/test/probe/test_object_failures.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import time from os import listdir, unlink from os.path import join as path_join from unittest import main, TestCase @@ -27,8 +28,22 @@ from swift.obj.diskfile import write_metadata, read_metadata from test.probe.common import kill_servers, reset_environment +RETRIES = 5 + + def get_data_file_path(obj_dir): - files = sorted(listdir(obj_dir), reverse=True) + files = [] + # We might need to try a few times if a request hasn't yet settled. For + # instance, a PUT can return success when just 2 of 3 nodes has completed. + for attempt in xrange(RETRIES + 1): + try: + files = sorted(listdir(obj_dir), reverse=True) + break + except Exception: + if attempt < RETRIES: + time.sleep(1) + else: + raise for filename in files: return path_join(obj_dir, filename)