ring: pickles now use only stdlib objects; old and really old pickles can still be read

bin/swift-ring-builder

@@ -19,7 +19,7 @@ from errno import EEXIST
 from gzip import GzipFile
 from os import mkdir
 from os.path import basename, dirname, exists, join as pathjoin
-from sys import argv, exit
+from sys import argv, exit, modules
 from textwrap import wrap
 from time import time
 
@@ -153,9 +153,9 @@ swift-ring-builder <builder_file> create <part_power> <replicas>
         except OSError, err:
             if err.errno != EEXIST:
                 raise
-        pickle.dump(builder, open(pathjoin(backup_dir,
+        pickle.dump(builder.to_dict(), open(pathjoin(backup_dir,
             '%d.' % time() + basename(argv[1])), 'wb'), protocol=2)
-        pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
+        pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_CHANGED)
 
     def default():
@@ -312,7 +312,7 @@ swift-ring-builder <builder_file> add z<zone>-<ip>:<port>/<device_name>_<meta>
         else:
             print 'Device z%s-%s:%s/%s_"%s" with %s weight got id %s' % \
                   (zone, ip, port, device_name, meta, weight, next_dev_id)
-        pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
+        pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_UNCHANGED)
 
     def set_weight():
@@ -345,7 +345,7 @@ swift-ring-builder <builder_file> set_weight <search-value> <weight>
             builder.set_dev_weight(dev['id'], weight)
             print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s" ' \
                   'weight set to %(weight)s' % dev
-        pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
+        pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_UNCHANGED)
 
     def set_info():
@@ -427,7 +427,7 @@ swift-ring-builder <builder_file> set_info <search-value>
             for key, value in change:
                 dev[key] = value
             print 'Device %s is now %s' % (orig_dev_string, format_device(dev))
-        pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
+        pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_UNCHANGED)
 
     def remove():
@@ -463,7 +463,7 @@ swift-ring-builder <builder_file> remove <search-value>
             print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s" ' \
                   'marked for removal and will be removed next rebalance.' \
                   % dev
-        pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
+        pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_UNCHANGED)
 
     def rebalance():
@@ -495,13 +495,14 @@ swift-ring-builder <builder_file> rebalance
                   % builder.min_part_hours
             print '-' * 79
         ts = time()
-        pickle.dump(builder.get_ring(),
+        pickle.dump(builder.get_ring().to_dict(),
                     GzipFile(pathjoin(backup_dir, '%d.' % ts +
                         basename(ring_file)), 'wb'), protocol=2)
-        pickle.dump(builder, open(pathjoin(backup_dir,
+        pickle.dump(builder.to_dict(), open(pathjoin(backup_dir,
             '%d.' % ts + basename(argv[1])), 'wb'), protocol=2)
-        pickle.dump(builder.get_ring(), GzipFile(ring_file, 'wb'), protocol=2)
+        pickle.dump(builder.get_ring().to_dict(), GzipFile(ring_file, 'wb'),
-        pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
+                    protocol=2)
+        pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_CHANGED)
 
     def validate():
@@ -528,15 +529,15 @@ swift-ring-builder <builder_file> write_ring
                       '"rebalance"?'
             else:
                 print 'Warning: Writing an empty ring'
-        pickle.dump(ring_data,
+        pickle.dump(ring_data.to_dict(),
                     GzipFile(pathjoin(backup_dir, '%d.' % time() +
                         basename(ring_file)), 'wb'), protocol=2)
-        pickle.dump(ring_data, GzipFile(ring_file, 'wb'), protocol=2)
+        pickle.dump(ring_data.to_dict(), GzipFile(ring_file, 'wb'), protocol=2)
         exit(EXIT_RING_CHANGED)
 
     def pretend_min_part_hours_passed():
         builder.pretend_min_part_hours_passed()
-        pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
+        pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_UNCHANGED)
 
     def set_min_part_hours():
@@ -552,7 +553,7 @@ swift-ring-builder <builder_file> set_min_part_hours <hours>
         builder.change_min_part_hours(int(argv[3]))
         print 'The minimum number of hours before a partition can be ' \
               'reassigned is now set to %s' % argv[3]
-        pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
+        pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
         exit(EXIT_RING_UNCHANGED)
 
 
@@ -578,7 +579,17 @@ if __name__ == '__main__':
         exit(EXIT_RING_UNCHANGED)
 
     if exists(argv[1]):
-        builder = pickle.load(open(argv[1], 'rb'))
+        try:
+            builder = pickle.load(open(argv[1], 'rb'))
+            if not hasattr(builder, 'devs'):
+                builder_dict = builder
+                builder = RingBuilder(1, 1, 1)
+                builder.copy_from(builder_dict)
+        except ImportError:  # Happens with really old builder pickles
+            modules['swift.ring_builder'] = \
+                modules['swift.common.ring.builder']
+            builder = RingBuilder(1, 1, 1)
+            builder.copy_from(pickle.load(open(argv[1], 'rb')))
         for dev in builder.devs:
             if dev and 'meta' not in dev:
                 dev['meta'] = ''

swift/common/ring/builder.py

@@ -69,6 +69,49 @@ class RingBuilder(object):
         self._remove_devs = []
         self._ring = None
 
+    def copy_from(self, builder):
+        if hasattr(builder, 'devs'):
+            self.part_power = builder.part_power
+            self.replicas = builder.replicas
+            self.min_part_hours = builder.min_part_hours
+            self.parts = builder.parts
+            self.devs = builder.devs
+            self.devs_changed = builder.devs_changed
+            self.version = builder.version
+            self._replica2part2dev = builder._replica2part2dev
+            self._last_part_moves_epoch = builder._last_part_moves_epoch
+            self._last_part_moves = builder._last_part_moves
+            self._last_part_gather_start = builder._last_part_gather_start
+            self._remove_devs = builder._remove_devs
+        else:
+            self.part_power = builder['part_power']
+            self.replicas = builder['replicas']
+            self.min_part_hours = builder['min_part_hours']
+            self.parts = builder['parts']
+            self.devs = builder['devs']
+            self.devs_changed = builder['devs_changed']
+            self.version = builder['version']
+            self._replica2part2dev = builder['_replica2part2dev']
+            self._last_part_moves_epoch = builder['_last_part_moves_epoch']
+            self._last_part_moves = builder['_last_part_moves']
+            self._last_part_gather_start = builder['_last_part_gather_start']
+            self._remove_devs = builder['_remove_devs']
+        self._ring = None
+
+    def to_dict(self):
+        return {'part_power': self.part_power,
+                'replicas': self.replicas,
+                'min_part_hours': self.min_part_hours,
+                'parts': self.parts,
+                'devs': self.devs,
+                'devs_changed': self.devs_changed,
+                'version': self.version,
+                '_replica2part2dev': self._replica2part2dev,
+                '_last_part_moves_epoch': self._last_part_moves_epoch,
+                '_last_part_moves': self._last_part_moves,
+                '_last_part_gather_start': self._last_part_gather_start,
+                '_remove_devs': self._remove_devs}
+
     def change_min_part_hours(self, min_part_hours):
         """
         Changes the value used to decide if a given partition can be moved

swift/common/ring/ring.py

@@ -29,6 +29,11 @@ class RingData(object):
         self._replica2part2dev_id = replica2part2dev_id
         self._part_shift = part_shift
 
+    def to_dict(self):
+        return {'devs': self.devs,
+                'replica2part2dev_id': self._replica2part2dev_id,
+                'part_shift': self._part_shift}
+
 
 class Ring(object):
     """
@@ -47,6 +52,9 @@ class Ring(object):
         self._rtime = time() + self.reload_time
         if force or self.has_changed():
             ring_data = pickle.load(GzipFile(self.pickle_gz_path, 'rb'))
+            if not hasattr(ring_data, 'devs'):
+                ring_data = RingData(ring_data['replica2part2dev_id'],
+                    ring_data['devs'], ring_data['part_shift'])
             self._mtime = getmtime(self.pickle_gz_path)
             self.devs = ring_data.devs
             self.zone2devs = {}