From 37e0654adb3563bc84176ebdea4e36f97e3c3bb5 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 29 Oct 2013 13:04:59 -0700 Subject: [PATCH] in case you lose your builder backups Change-Id: Ica555be2be492c3ec5fdeab738058ff35989a603 --- bin/swift-ring-builder | 63 ++++++++++++++++++++++++----- swift/common/ring/utils.py | 16 ++++++++ test/unit/common/ring/test_utils.py | 22 +++++++++- 3 files changed, 90 insertions(+), 11 deletions(-) diff --git a/bin/swift-ring-builder b/bin/swift-ring-builder index 3bd804a028..4eaa732b78 100755 --- a/bin/swift-ring-builder +++ b/bin/swift-ring-builder @@ -25,11 +25,11 @@ from textwrap import wrap from time import time from swift.common import exceptions -from swift.common.ring import RingBuilder +from swift.common.ring import RingBuilder, Ring from swift.common.ring.builder import MAX_BALANCE from swift.common.utils import lock_parent_directory from swift.common.ring.utils import parse_search_value, parse_args, \ - build_dev_from_opts + build_dev_from_opts, parse_builder_ring_filename_args MAJOR_VERSION = 1 MINOR_VERSION = 3 @@ -697,6 +697,52 @@ swift-ring-builder write_ring ring_data.save(ring_file) exit(EXIT_SUCCESS) + def write_builder(): + """ +swift-ring-builder write_builder [min_part_hours] + Recreate a builder from a ring file (lossy) if you lost your builder + backups. (Protip: don't lose your builder backups). + [min_part_hours] is one of those numbers lost to the builder, + you can change it with set_min_part_hours. + """ + if exists(builder_file): + print 'Cowardly refusing to overwrite existing ' \ + 'Ring Builder file: %s' % builder_file + exit(EXIT_ERROR) + if len(argv) > 3: + min_part_hours = int(argv[3]) + else: + stderr.write("WARNING: default min_part_hours may not match " + "the value in the lost builder.\n") + min_part_hours = 24 + ring = Ring(ring_file) + for dev in ring.devs: + dev.update({ + 'parts': 0, + 'parts_wanted': 0, + }) + builder_dict = { + 'part_power': 32 - ring._part_shift, + 'replicas': float(ring.replica_count), + 'min_part_hours': min_part_hours, + 'parts': ring.partition_count, + 'devs': ring.devs, + 'devs_changed': False, + 'version': 0, + '_replica2part2dev': ring._replica2part2dev_id, + '_last_part_moves_epoch': None, + '_last_part_moves': None, + '_last_part_gather_start': 0, + '_remove_devs': [], + } + builder = RingBuilder(1, 1, 1) + builder.copy_from(builder_dict) + for parts in builder._replica2part2dev: + for dev_id in parts: + builder.devs[dev_id]['parts'] += 1 + builder._set_parts_wanted() + builder.save(builder_file) + def pretend_min_part_hours_passed(): builder.pretend_min_part_hours_passed() builder.save(argv[1]) @@ -772,9 +818,11 @@ if __name__ == '__main__': ' 2 = error') exit(EXIT_SUCCESS) - if exists(argv[1]): - builder = RingBuilder.load(argv[1]) - elif len(argv) < 3 or argv[2] != 'create': + builder_file, ring_file = parse_builder_ring_filename_args(argv) + + if exists(builder_file): + builder = RingBuilder.load(builder_file) + elif len(argv) < 3 or argv[2] not in('create', 'write_builder'): print 'Ring Builder file does not exist: %s' % argv[1] exit(EXIT_ERROR) @@ -785,11 +833,6 @@ if __name__ == '__main__': if err.errno != EEXIST: raise - ring_file = argv[1] - if ring_file.endswith('.builder'): - ring_file = ring_file[:-len('.builder')] - ring_file += '.ring.gz' - if len(argv) == 2: command = "default" else: diff --git a/swift/common/ring/utils.py b/swift/common/ring/utils.py index 5ab7e58da8..45899041b7 100644 --- a/swift/common/ring/utils.py +++ b/swift/common/ring/utils.py @@ -269,6 +269,22 @@ def parse_args(argvish): return parser.parse_args(argvish) +def parse_builder_ring_filename_args(argvish): + first_arg = argvish[1] + if first_arg.endswith('.ring.gz'): + ring_file = first_arg + builder_file = first_arg[:-len('.ring.gz')] + '.builder' + else: + builder_file = first_arg + if not builder_file.endswith('.builder'): + ring_file = first_arg + else: + ring_file = builder_file[:-len('.builder')] + if not first_arg.endswith('.ring.gz'): + ring_file += '.ring.gz' + return builder_file, ring_file + + def build_dev_from_opts(opts): """ Convert optparse stype options into a device dictionary. diff --git a/test/unit/common/ring/test_utils.py b/test/unit/common/ring/test_utils.py index da2f43e3e4..80c695c819 100644 --- a/test/unit/common/ring/test_utils.py +++ b/test/unit/common/ring/test_utils.py @@ -17,7 +17,8 @@ import unittest from swift.common.ring.utils import (build_tier_tree, tiers_for_dev, parse_search_value, parse_args, - build_dev_from_opts) + build_dev_from_opts, + parse_builder_ring_filename_args) class TestUtils(unittest.TestCase): @@ -139,6 +140,25 @@ class TestUtils(unittest.TestCase): } self.assertEquals(device, expected) + def test_parse_builder_ring_filename_args(self): + args = 'swift-ring-builder object.builder write_ring' + self.assertEquals(( + 'object.builder', 'object.ring.gz' + ), parse_builder_ring_filename_args(args.split())) + args = 'swift-ring-builder container.ring.gz write_builder' + self.assertEquals(( + 'container.builder', 'container.ring.gz' + ), parse_builder_ring_filename_args(args.split())) + # builer name arg should always fall through + args = 'swift-ring-builder test create' + self.assertEquals(( + 'test', 'test.ring.gz' + ), parse_builder_ring_filename_args(args.split())) + args = 'swift-ring-builder my.file.name create' + self.assertEquals(( + 'my.file.name', 'my.file.name.ring.gz' + ), parse_builder_ring_filename_args(args.split())) + if __name__ == '__main__': unittest.main()