diff --git a/swift/cli/info.py b/swift/cli/info.py index 7826a17b81..d99fb3b193 100644 --- a/swift/cli/info.py +++ b/swift/cli/info.py @@ -30,6 +30,7 @@ from swift.container.backend import ContainerBroker, DATADIR as CBDATADIR from swift.obj.diskfile import get_data_dir, read_metadata, DATADIR_BASE, \ extract_policy from swift.common.storage_policy import POLICIES +from swift.common.swob import wsgi_to_str from swift.common.middleware.crypto.crypto_utils import load_crypto_meta from swift.common.utils import md5 @@ -537,6 +538,8 @@ def print_obj(datafile, check_etag=True, swift_dir='/etc/swift', except EOFError: print("Invalid metadata") raise InfoSystemExit() + metadata = {wsgi_to_str(k): v if k == 'name' else wsgi_to_str(v) + for k, v in metadata.items()} etag = metadata.pop('ETag', '') length = metadata.pop('Content-Length', '') diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py index efd8979077..8b6b7d01b4 100644 --- a/swift/obj/diskfile.py +++ b/swift/obj/diskfile.py @@ -167,24 +167,36 @@ def _encode_metadata(metadata): return dict(((encode_str(k), encode_str(v)) for k, v in metadata.items())) -def _decode_metadata(metadata): +def _decode_metadata(metadata, metadata_written_by_py3): """ Given a metadata dict from disk, convert keys and values to native strings. :param metadata: a dict + :param metadata_written_by_py3: """ if six.PY2: - def to_str(item): + def to_str(item, is_name=False): + # For years, py2 and py3 handled non-ascii metadata differently; + # see https://bugs.launchpad.net/swift/+bug/2012531 + if metadata_written_by_py3 and not is_name: + # do our best to read new-style data replicated from a py3 node + item = item.decode('utf8').encode('latin1') if isinstance(item, six.text_type): return item.encode('utf8') return item else: - def to_str(item): + def to_str(item, is_name=False): + # For years, py2 and py3 handled non-ascii metadata differently; + # see https://bugs.launchpad.net/swift/+bug/2012531 + if not metadata_written_by_py3 and isinstance(item, bytes) \ + and not is_name: + # do our best to read old py2 data + item = item.decode('latin1') if isinstance(item, six.binary_type): return item.decode('utf8', 'surrogateescape') return item - return dict(((to_str(k), to_str(v)) for k, v in metadata.items())) + return {to_str(k): to_str(v, k == b'name') for k, v in metadata.items()} def read_metadata(fd, add_missing_checksum=False): @@ -238,6 +250,7 @@ def read_metadata(fd, add_missing_checksum=False): "stored checksum='%s', computed='%s'" % ( fd, metadata_checksum, computed_checksum)) + metadata_written_by_py3 = (b'_codecs\nencode' in metadata[:32]) # strings are utf-8 encoded when written, but have not always been # (see https://bugs.launchpad.net/swift/+bug/1678018) so encode them again # when read @@ -245,7 +258,7 @@ def read_metadata(fd, add_missing_checksum=False): metadata = pickle.loads(metadata) else: metadata = pickle.loads(metadata, encoding='bytes') - return _decode_metadata(metadata) + return _decode_metadata(metadata, metadata_written_by_py3) def write_metadata(fd, metadata, xattr_size=65536): diff --git a/test/unit/cli/test_info.py b/test/unit/cli/test_info.py index 80813db87d..85fb79819e 100644 --- a/test/unit/cli/test_info.py +++ b/test/unit/cli/test_info.py @@ -1219,7 +1219,7 @@ class TestPrintObjFullMeta(TestCliInfoBase): md = {'name': '/AUTH_admin/c/obj', 'Content-Type': 'application/octet-stream', 'ETag': 'd41d8cd98f00b204e9800998ecf8427e', - 'Content-Length': 0} + 'Content-Length': '0'} write_metadata(fp, md) def test_print_obj(self): @@ -1256,7 +1256,7 @@ class TestPrintObjFullMeta(TestCliInfoBase): md = {'name': '/AUTH_admin/c/obj', 'Content-Type': 'application/octet-stream', 'ETag': 'd41d8cd98f00b204e9800998ecf8427e', - 'Content-Length': 0} + 'Content-Length': '0'} write_metadata(fp, md) object_ring = ring.Ring(self.testdir, ring_name='object-2') @@ -1297,7 +1297,7 @@ class TestPrintObjFullMeta(TestCliInfoBase): md = {'name': '/AUTH_admin/c/obj', 'Content-Type': 'application/octet-stream', 'ETag': 'd41d8cd98f00b204e9800998ecf8427e', - 'Content-Length': 0} + 'Content-Length': '0'} write_metadata(fp, md) object_ring = ring.Ring(self.testdir, ring_name='object-3') @@ -1368,7 +1368,7 @@ class TestPrintObjFullMeta(TestCliInfoBase): md = {'name': '/AUTH_admin/c/obj', 'Content-Type': 'application/octet-stream', 'ETag': 'badetag', - 'Content-Length': 0} + 'Content-Length': '0'} write_metadata(fp, md) out = StringIO() @@ -1703,5 +1703,5 @@ class TestPrintObjWeirdPath(TestPrintObjFullMeta): md = {'name': '/AUTH_admin/c/obj', 'Content-Type': 'application/octet-stream', 'ETag': 'd41d8cd98f00b204e9800998ecf8427e', - 'Content-Length': 0} + 'Content-Length': '0'} write_metadata(fp, md) diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index 327d860e51..8ee4edb457 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -15,6 +15,8 @@ """Tests for swift.obj.diskfile""" +from __future__ import print_function +import binascii import six.moves.cPickle as pickle import os import errno @@ -28,6 +30,7 @@ import uuid import xattr import re import six +import sys from collections import defaultdict from random import shuffle, randint from shutil import rmtree @@ -554,6 +557,233 @@ class TestDiskFileModuleMethods(unittest.TestCase): exp_dir = '/srv/node/sdb5/objects-1/123' self.assertEqual(part_dir, exp_dir) + def test_can_read_old_meta(self): + # outputs taken from `xattr -l ` + cases = { + 'python_2.7.18_swift_2.13_replicated': ''' +0000 80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D ..}q.(U.Content- +0010 4C 65 6E 67 74 68 71 02 55 02 31 33 55 04 6E 61 Lengthq.U.13U.na +0020 6D 65 71 03 55 12 2F 41 55 54 48 5F 74 65 73 74 meq.U./AUTH_test +0030 2F E2 98 83 2F E2 98 83 71 04 55 13 58 2D 4F 62 /.../...q.U.X-Ob +0040 6A 65 63 74 2D 4D 65 74 61 2D 4D 74 69 6D 65 55 ject-Meta-MtimeU +0050 11 31 36 38 32 39 35 39 38 37 34 2E 37 35 36 32 .1682959874.7562 +0060 30 35 71 05 55 04 45 54 61 67 71 06 55 20 36 62 05q.U.ETagq.U 6b +0070 37 64 39 61 31 63 35 64 31 36 37 63 63 35 30 30 7d9a1c5d167cc500 +0080 33 37 66 32 39 66 32 39 30 62 62 33 37 35 71 07 37f29f290bb375q. +0090 55 0B 58 2D 54 69 6D 65 73 74 61 6D 70 71 08 55 U.X-Timestampq.U +00A0 10 31 36 38 32 39 36 32 36 35 31 2E 39 37 34 39 .1682962651.9749 +00B0 34 55 11 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 61 4U.X-Object-Meta +00C0 2D E2 98 83 55 03 E2 98 83 71 09 55 0C 43 6F 6E -...U....q.U.Con +00D0 74 65 6E 74 2D 54 79 70 65 71 0A 55 18 61 70 70 tent-Typeq.U.app +00E0 6C 69 63 61 74 69 6F 6E 2F 6F 63 74 65 74 2D 73 lication/octet-s +00F0 74 72 65 61 6D 71 0B 75 2E treamq.u. + ''', + 'python_2.7.18_swift_2.13_ec': ''' +0000 80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D ..}q.(U.Content- +0010 4C 65 6E 67 74 68 71 02 55 02 38 34 55 04 6E 61 Lengthq.U.84U.na +0020 6D 65 71 03 55 12 2F 41 55 54 48 5F 74 65 73 74 meq.U./AUTH_test +0030 2F E2 98 83 2F E2 98 83 71 04 58 1E 00 00 00 58 /.../...q.X....X +0040 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 2D -Object-Sysmeta- +0050 45 63 2D 46 72 61 67 2D 49 6E 64 65 78 71 05 55 Ec-Frag-Indexq.U +0060 01 35 55 13 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 .5U.X-Object-Met +0070 61 2D 4D 74 69 6D 65 55 11 31 36 38 32 39 35 39 a-MtimeU.1682959 +0080 38 37 34 2E 37 35 36 32 30 35 71 06 58 22 00 00 874.756205q.X".. +0090 00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 .X-Object-Sysmet +00A0 61 2D 45 63 2D 43 6F 6E 74 65 6E 74 2D 4C 65 6E a-Ec-Content-Len +00B0 67 74 68 71 07 55 02 31 33 71 08 58 18 00 00 00 gthq.U.13q.X.... +00C0 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 X-Object-Sysmeta +00D0 2D 45 63 2D 45 74 61 67 71 09 55 20 36 62 37 64 -Ec-Etagq.U 6b7d +00E0 39 61 31 63 35 64 31 36 37 63 63 35 30 30 33 37 9a1c5d167cc50037 +00F0 66 32 39 66 32 39 30 62 62 33 37 35 71 0A 55 04 f29f290bb375q.U. +0100 45 54 61 67 71 0B 55 20 65 32 66 64 34 33 30 65 ETagq.U e2fd430e +0110 61 66 37 32 32 33 63 32 35 30 33 63 34 65 38 33 af7223c2503c4e83 +0120 30 31 63 66 66 33 37 63 71 0C 55 0B 58 2D 54 69 01cff37cq.U.X-Ti +0130 6D 65 73 74 61 6D 70 71 0D 55 10 31 36 38 32 39 mestampq.U.16829 +0140 36 32 32 36 32 2E 31 36 31 39 39 55 11 58 2D 4F 62262.16199U.X-O +0150 62 6A 65 63 74 2D 4D 65 74 61 2D E2 98 83 55 03 bject-Meta-...U. +0160 E2 98 83 71 0E 58 1A 00 00 00 58 2D 4F 62 6A 65 ...q.X....X-Obje +0170 63 74 2D 53 79 73 6D 65 74 61 2D 45 63 2D 53 63 ct-Sysmeta-Ec-Sc +0180 68 65 6D 65 71 0F 55 1A 6C 69 62 65 72 61 73 75 hemeq.U.liberasu +0190 72 65 63 6F 64 65 5F 72 73 5F 76 61 6E 64 20 34 recode_rs_vand 4 +01A0 2B 32 71 10 55 0C 43 6F 6E 74 65 6E 74 2D 54 79 +2q.U.Content-Ty +01B0 70 65 71 11 55 18 61 70 70 6C 69 63 61 74 69 6F peq.U.applicatio +01C0 6E 2F 6F 63 74 65 74 2D 73 74 72 65 61 6D 71 12 n/octet-streamq. +01D0 58 20 00 00 00 58 2D 4F 62 6A 65 63 74 2D 53 79 X ...X-Object-Sy +01E0 73 6D 65 74 61 2D 45 63 2D 53 65 67 6D 65 6E 74 smeta-Ec-Segment +01F0 2D 53 69 7A 65 71 13 55 07 31 30 34 38 35 37 36 -Sizeq.U.1048576 +0200 71 14 75 2E q.u. + ''', + 'python_2.7.18_swift_2.23_replicated': ''' +0000 80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D ..}q.(U.Content- +0010 4C 65 6E 67 74 68 71 02 55 02 31 33 71 03 55 04 Lengthq.U.13q.U. +0020 6E 61 6D 65 71 04 55 12 2F 41 55 54 48 5F 74 65 nameq.U./AUTH_te +0030 73 74 2F E2 98 83 2F E2 98 83 71 05 55 0C 43 6F st/.../...q.U.Co +0040 6E 74 65 6E 74 2D 54 79 70 65 71 06 55 18 61 70 ntent-Typeq.U.ap +0050 70 6C 69 63 61 74 69 6F 6E 2F 6F 63 74 65 74 2D plication/octet- +0060 73 74 72 65 61 6D 71 07 55 04 45 54 61 67 71 08 streamq.U.ETagq. +0070 55 20 36 62 37 64 39 61 31 63 35 64 31 36 37 63 U 6b7d9a1c5d167c +0080 63 35 30 30 33 37 66 32 39 66 32 39 30 62 62 33 c50037f29f290bb3 +0090 37 35 71 09 55 0B 58 2D 54 69 6D 65 73 74 61 6D 75q.U.X-Timestam +00A0 70 71 0A 55 10 31 36 38 32 39 36 33 32 30 39 2E pq.U.1682963209. +00B0 38 32 32 37 32 71 0B 55 11 58 2D 4F 62 6A 65 63 82272q.U.X-Objec +00C0 74 2D 4D 65 74 61 2D E2 98 83 71 0C 55 03 E2 98 t-Meta-...q.U... +00D0 83 71 0D 55 13 58 2D 4F 62 6A 65 63 74 2D 4D 65 .q.U.X-Object-Me +00E0 74 61 2D 4D 74 69 6D 65 71 0E 55 11 31 36 38 32 ta-Mtimeq.U.1682 +00F0 39 35 39 38 37 34 2E 37 35 36 32 30 35 71 0F 75 959874.756205q.u +0100 2E . + ''', + 'python_3.10.6_swift_2.23_replicated': ''' +0000 80 02 7D 71 00 28 63 5F 63 6F 64 65 63 73 0A 65 ..}q.(c_codecs.e +0010 6E 63 6F 64 65 0A 71 01 58 0B 00 00 00 58 2D 54 ncode.q.X....X-T +0020 69 6D 65 73 74 61 6D 70 71 02 58 06 00 00 00 6C imestampq.X....l +0030 61 74 69 6E 31 71 03 86 71 04 52 71 05 68 01 58 atin1q..q.Rq.h.X +0040 10 00 00 00 31 36 38 32 39 36 33 30 31 37 2E 31 ....1682963017.1 +0050 30 34 37 32 71 06 68 03 86 71 07 52 71 08 68 01 0472q.h..q.Rq.h. +0060 58 0C 00 00 00 43 6F 6E 74 65 6E 74 2D 54 79 70 X....Content-Typ +0070 65 71 09 68 03 86 71 0A 52 71 0B 68 01 58 18 00 eq.h..q.Rq.h.X.. +0080 00 00 61 70 70 6C 69 63 61 74 69 6F 6E 2F 6F 63 ..application/oc +0090 74 65 74 2D 73 74 72 65 61 6D 71 0C 68 03 86 71 tet-streamq.h..q +00A0 0D 52 71 0E 68 01 58 0E 00 00 00 43 6F 6E 74 65 .Rq.h.X....Conte +00B0 6E 74 2D 4C 65 6E 67 74 68 71 0F 68 03 86 71 10 nt-Lengthq.h..q. +00C0 52 71 11 68 01 58 02 00 00 00 31 33 71 12 68 03 Rq.h.X....13q.h. +00D0 86 71 13 52 71 14 68 01 58 04 00 00 00 45 54 61 .q.Rq.h.X....ETa +00E0 67 71 15 68 03 86 71 16 52 71 17 68 01 58 20 00 gq.h..q.Rq.h.X . +00F0 00 00 36 62 37 64 39 61 31 63 35 64 31 36 37 63 ..6b7d9a1c5d167c +0100 63 35 30 30 33 37 66 32 39 66 32 39 30 62 62 33 c50037f29f290bb3 +0110 37 35 71 18 68 03 86 71 19 52 71 1A 68 01 58 13 75q.h..q.Rq.h.X. +0120 00 00 00 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 61 ...X-Object-Meta +0130 2D 4D 74 69 6D 65 71 1B 68 03 86 71 1C 52 71 1D -Mtimeq.h..q.Rq. +0140 68 01 58 11 00 00 00 31 36 38 32 39 35 39 38 37 h.X....168295987 +0150 34 2E 37 35 36 32 30 35 71 1E 68 03 86 71 1F 52 4.756205q.h..q.R +0160 71 20 68 01 58 1A 00 00 00 58 2D 4F 62 6A 65 63 q h.X....X-Objec +0170 74 2D 4D 65 74 61 2D C3 83 C2 A2 C3 82 C2 98 C3 t-Meta-......... +0180 82 C2 83 71 21 68 03 86 71 22 52 71 23 68 01 58 ...q!h..q"Rq#h.X +0190 0C 00 00 00 C3 83 C2 A2 C3 82 C2 98 C3 82 C2 83 ................ +01A0 71 24 68 03 86 71 25 52 71 26 68 01 58 04 00 00 q$h..q%Rq&h.X... +01B0 00 6E 61 6D 65 71 27 68 03 86 71 28 52 71 29 68 .nameq'h..q(Rq)h +01C0 01 58 18 00 00 00 2F 41 55 54 48 5F 74 65 73 74 .X..../AUTH_test +01D0 2F C3 A2 C2 98 C2 83 2F C3 A2 C2 98 C2 83 71 2A /....../......q* +01E0 68 03 86 71 2B 52 71 2C 75 2E h..q+Rq,u. + ''', + 'python_2.7.18_swift_2.23_ec': ''' +0000 80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D ..}q.(U.Content- +0010 4C 65 6E 67 74 68 71 02 55 02 38 34 71 03 55 04 Lengthq.U.84q.U. +0020 6E 61 6D 65 71 04 55 12 2F 41 55 54 48 5F 74 65 nameq.U./AUTH_te +0030 73 74 2F E2 98 83 2F E2 98 83 71 05 55 1E 58 2D st/.../...q.U.X- +0040 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 2D 45 Object-Sysmeta-E +0050 63 2D 46 72 61 67 2D 49 6E 64 65 78 55 01 35 55 c-Frag-IndexU.5U +0060 0C 43 6F 6E 74 65 6E 74 2D 54 79 70 65 71 06 55 .Content-Typeq.U +0070 18 61 70 70 6C 69 63 61 74 69 6F 6E 2F 6F 63 74 .application/oct +0080 65 74 2D 73 74 72 65 61 6D 71 07 55 22 58 2D 4F et-streamq.U"X-O +0090 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 2D 45 63 bject-Sysmeta-Ec +00A0 2D 43 6F 6E 74 65 6E 74 2D 4C 65 6E 67 74 68 55 -Content-LengthU +00B0 02 31 33 71 08 55 18 58 2D 4F 62 6A 65 63 74 2D .13q.U.X-Object- +00C0 53 79 73 6D 65 74 61 2D 45 63 2D 45 74 61 67 55 Sysmeta-Ec-EtagU +00D0 20 36 62 37 64 39 61 31 63 35 64 31 36 37 63 63 6b7d9a1c5d167cc +00E0 35 30 30 33 37 66 32 39 66 32 39 30 62 62 33 37 50037f29f290bb37 +00F0 35 71 09 55 04 45 54 61 67 71 0A 55 20 65 32 66 5q.U.ETagq.U e2f +0100 64 34 33 30 65 61 66 37 32 32 33 63 32 35 30 33 d430eaf7223c2503 +0110 63 34 65 38 33 30 31 63 66 66 33 37 63 71 0B 55 c4e8301cff37cq.U +0120 0B 58 2D 54 69 6D 65 73 74 61 6D 70 71 0C 55 10 .X-Timestampq.U. +0130 31 36 38 32 39 36 33 31 33 30 2E 33 35 39 38 36 1682963130.35986 +0140 71 0D 55 11 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 q.U.X-Object-Met +0150 61 2D E2 98 83 71 0E 55 03 E2 98 83 71 0F 55 1A a-...q.U....q.U. +0160 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 X-Object-Sysmeta +0170 2D 45 63 2D 53 63 68 65 6D 65 55 1A 6C 69 62 65 -Ec-SchemeU.libe +0180 72 61 73 75 72 65 63 6F 64 65 5F 72 73 5F 76 61 rasurecode_rs_va +0190 6E 64 20 34 2B 32 71 10 55 13 58 2D 4F 62 6A 65 nd 4+2q.U.X-Obje +01A0 63 74 2D 4D 65 74 61 2D 4D 74 69 6D 65 71 11 55 ct-Meta-Mtimeq.U +01B0 11 31 36 38 32 39 35 39 38 37 34 2E 37 35 36 32 .1682959874.7562 +01C0 30 35 71 12 55 20 58 2D 4F 62 6A 65 63 74 2D 53 05q.U X-Object-S +01D0 79 73 6D 65 74 61 2D 45 63 2D 53 65 67 6D 65 6E ysmeta-Ec-Segmen +01E0 74 2D 53 69 7A 65 55 07 31 30 34 38 35 37 36 71 t-SizeU.1048576q +01F0 13 75 2E .u. + ''', + 'python_3.10.6_swift_2.23_ec': ''' +0000 80 02 7D 71 00 28 63 5F 63 6F 64 65 63 73 0A 65 ..}q.(c_codecs.e +0010 6E 63 6F 64 65 0A 71 01 58 0B 00 00 00 58 2D 54 ncode.q.X....X-T +0020 69 6D 65 73 74 61 6D 70 71 02 58 06 00 00 00 6C imestampq.X....l +0030 61 74 69 6E 31 71 03 86 71 04 52 71 05 68 01 58 atin1q..q.Rq.h.X +0040 10 00 00 00 31 36 38 32 39 36 32 39 35 35 2E 33 ....1682962955.3 +0050 37 35 34 36 71 06 68 03 86 71 07 52 71 08 68 01 7546q.h..q.Rq.h. +0060 58 0C 00 00 00 43 6F 6E 74 65 6E 74 2D 54 79 70 X....Content-Typ +0070 65 71 09 68 03 86 71 0A 52 71 0B 68 01 58 18 00 eq.h..q.Rq.h.X.. +0080 00 00 61 70 70 6C 69 63 61 74 69 6F 6E 2F 6F 63 ..application/oc +0090 74 65 74 2D 73 74 72 65 61 6D 71 0C 68 03 86 71 tet-streamq.h..q +00A0 0D 52 71 0E 68 01 58 0E 00 00 00 43 6F 6E 74 65 .Rq.h.X....Conte +00B0 6E 74 2D 4C 65 6E 67 74 68 71 0F 68 03 86 71 10 nt-Lengthq.h..q. +00C0 52 71 11 68 01 58 02 00 00 00 38 34 71 12 68 03 Rq.h.X....84q.h. +00D0 86 71 13 52 71 14 68 01 58 04 00 00 00 45 54 61 .q.Rq.h.X....ETa +00E0 67 71 15 68 03 86 71 16 52 71 17 68 01 58 20 00 gq.h..q.Rq.h.X . +00F0 00 00 65 32 66 64 34 33 30 65 61 66 37 32 32 33 ..e2fd430eaf7223 +0100 63 32 35 30 33 63 34 65 38 33 30 31 63 66 66 33 c2503c4e8301cff3 +0110 37 63 71 18 68 03 86 71 19 52 71 1A 68 01 58 13 7cq.h..q.Rq.h.X. +0120 00 00 00 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 61 ...X-Object-Meta +0130 2D 4D 74 69 6D 65 71 1B 68 03 86 71 1C 52 71 1D -Mtimeq.h..q.Rq. +0140 68 01 58 11 00 00 00 31 36 38 32 39 35 39 38 37 h.X....168295987 +0150 34 2E 37 35 36 32 30 35 71 1E 68 03 86 71 1F 52 4.756205q.h..q.R +0160 71 20 68 01 58 1A 00 00 00 58 2D 4F 62 6A 65 63 q h.X....X-Objec +0170 74 2D 4D 65 74 61 2D C3 83 C2 A2 C3 82 C2 98 C3 t-Meta-......... +0180 82 C2 83 71 21 68 03 86 71 22 52 71 23 68 01 58 ...q!h..q"Rq#h.X +0190 0C 00 00 00 C3 83 C2 A2 C3 82 C2 98 C3 82 C2 83 ................ +01A0 71 24 68 03 86 71 25 52 71 26 68 01 58 18 00 00 q$h..q%Rq&h.X... +01B0 00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 .X-Object-Sysmet +01C0 61 2D 45 63 2D 45 74 61 67 71 27 68 03 86 71 28 a-Ec-Etagq'h..q( +01D0 52 71 29 68 01 58 20 00 00 00 36 62 37 64 39 61 Rq)h.X ...6b7d9a +01E0 31 63 35 64 31 36 37 63 63 35 30 30 33 37 66 32 1c5d167cc50037f2 +01F0 39 66 32 39 30 62 62 33 37 35 71 2A 68 03 86 71 9f290bb375q*h..q +0200 2B 52 71 2C 68 01 58 22 00 00 00 58 2D 4F 62 6A +Rq,h.X"...X-Obj +0210 65 63 74 2D 53 79 73 6D 65 74 61 2D 45 63 2D 43 ect-Sysmeta-Ec-C +0220 6F 6E 74 65 6E 74 2D 4C 65 6E 67 74 68 71 2D 68 ontent-Lengthq-h +0230 03 86 71 2E 52 71 2F 68 01 58 02 00 00 00 31 33 ..q.Rq/h.X....13 +0240 71 30 68 03 86 71 31 52 71 32 68 01 58 1E 00 00 q0h..q1Rq2h.X... +0250 00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 .X-Object-Sysmet +0260 61 2D 45 63 2D 46 72 61 67 2D 49 6E 64 65 78 71 a-Ec-Frag-Indexq +0270 33 68 03 86 71 34 52 71 35 68 01 58 01 00 00 00 3h..q4Rq5h.X.... +0280 35 71 36 68 03 86 71 37 52 71 38 68 01 58 1A 00 5q6h..q7Rq8h.X.. +0290 00 00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 ..X-Object-Sysme +02A0 74 61 2D 45 63 2D 53 63 68 65 6D 65 71 39 68 03 ta-Ec-Schemeq9h. +02B0 86 71 3A 52 71 3B 68 01 58 1A 00 00 00 6C 69 62 .q:Rq;h.X....lib +02C0 65 72 61 73 75 72 65 63 6F 64 65 5F 72 73 5F 76 erasurecode_rs_v +02D0 61 6E 64 20 34 2B 32 71 3C 68 03 86 71 3D 52 71 and 4+2qh.X ...X-Object +02F0 2D 53 79 73 6D 65 74 61 2D 45 63 2D 53 65 67 6D -Sysmeta-Ec-Segm +0300 65 6E 74 2D 53 69 7A 65 71 3F 68 03 86 71 40 52 ent-Sizeq?h..q@R +0310 71 41 68 01 58 07 00 00 00 31 30 34 38 35 37 36 qAh.X....1048576 +0320 71 42 68 03 86 71 43 52 71 44 68 01 58 04 00 00 qBh..qCRqDh.X... +0330 00 6E 61 6D 65 71 45 68 03 86 71 46 52 71 47 68 .nameqEh..qFRqGh +0340 01 58 18 00 00 00 2F 41 55 54 48 5F 74 65 73 74 .X..../AUTH_test +0350 2F C3 A2 C2 98 C2 83 2F C3 A2 C2 98 C2 83 71 48 /....../......qH +0360 68 03 86 71 49 52 71 4A 75 2E h..qIRqJu. + ''', + } + + def raw_xattr(output): + return binascii.unhexlify(''.join( + line[7:55] for line in output.split('\n') + ).replace(' ', '')) + + path = os.path.join(self.testdir, str(uuid.uuid4())) + for case, xattr_output in cases.items(): + try: + to_write = raw_xattr(xattr_output) + with open(path, 'wb') as fp: + xattr.setxattr( + fp.fileno(), 'user.swift.metadata', to_write) + with open(path, 'rb') as fd: + actual = diskfile.read_metadata(fd) + # name should come out as native strings + expected_name = b'/AUTH_test/\xe2\x98\x83/\xe2\x98\x83' + if not six.PY2: + expected_name = expected_name.decode('utf8') + self.assertEqual(actual['name'], expected_name) + # other meta will be WSGI strings, though + self.assertEqual( + actual['X-Object-Meta-\xe2\x98\x83'], '\xe2\x98\x83') + except Exception: + print('Failure in %s' % case, file=sys.stderr) + raise + def test_write_read_metadata(self): path = os.path.join(self.testdir, str(uuid.uuid4())) metadata = {'name': '/a/c/o', @@ -597,29 +827,6 @@ class TestDiskFileModuleMethods(unittest.TestCase): diskfile.write_metadata(fd, metadata) check_metadata(as_native, str) - # mock the read path to check the write path encoded persisted metadata - with mock.patch.object(diskfile, '_decode_metadata', lambda x: x): - check_metadata(as_bytes, bytes) - - # simulate a legacy diskfile that might have persisted - # (some) unicode metadata - with mock.patch.object(diskfile, '_encode_metadata', lambda x: x): - with open(path, 'wb') as fd: - diskfile.write_metadata(fd, metadata) - # sanity check: mock read path again to see that we did persist unicode - with mock.patch.object(diskfile, '_decode_metadata', lambda x: x): - with open(path, 'rb') as fd: - actual = diskfile.read_metadata(fd) - for k, v in actual.items(): - if isinstance(k, six.text_type) and \ - k == u'X-Object-Meta-Strange': - self.assertIsInstance(v, six.text_type) - break - else: - self.fail('Did not find X-Object-Meta-Strange') - # check that read_metadata converts binary_type - check_metadata(as_native, str) - @patch_policies class TestObjectAuditLocationGenerator(unittest.TestCase):