Merge "blueprint Multi-range support implementation"

swift/common/swob.py

@@ -30,6 +30,7 @@ from email.utils import parsedate
 import urlparse
 import urllib2
 import re
+import random
 
 from swift.common.utils import reiterate
 
@@ -440,6 +441,22 @@ class Range(object):
     invalid byte-range-spec values MUST ignore the header field that includes
     that byte-range-set."
 
+    According to the RFC 2616 specification, the following cases will be all
+    considered as syntactically invalid, thus, a ValueError is thrown so that
+    the range header will be ignored. If the range value contains at least
+    one of the following cases, the entire range is considered invalid,
+    ValueError will be thrown so that the header will be ignored.
+
+    1. value not starts with bytes=
+    2. range value start is greater than the end, eg. bytes=5-3
+    3. range does not have start or end, eg. bytes=-
+    4. range does not have hyphen, eg. bytes=45
+    5. range value is non numeric
+    6. any combination of the above
+
+    Every syntactically valid range will be added into the ranges list
+    even when some of the ranges may not be satisfied by underlying content.
+
     :param headerval: value of the header as a str
     """
     def __init__(self, headerval):
@@ -448,22 +465,26 @@ class Range(object):
             raise ValueError('Invalid Range header: %s' % headerval)
         self.ranges = []
         for rng in headerval[6:].split(','):
+            # Check if the range has required hyphen.
+            if rng.find('-') == -1:
+                raise ValueError('Invalid Range header: %s' % headerval)
             start, end = rng.split('-', 1)
             if start:
+                # when start contains non numeric value, this also causes
+                # ValueError
                 start = int(start)
             else:
                 start = None
             if end:
+                # when end contains non numeric value, this also causes
+                # ValueError
                 end = int(end)
-                if start is not None and not end >= start:
-                    # If the last-byte-pos value is present, it MUST be greater
-                    # than or equal to the first-byte-pos in that
-                    # byte-range-spec, or the byte- range-spec is syntactically
-                    # invalid.  [which "MUST" be ignored]
-                    self.ranges = []
-                    break
+                if start is not None and end < start:
+                    raise ValueError('Invalid Range header: %s' % headerval)
             else:
                 end = None
+                if start is None:
+                    raise ValueError('Invalid Range header: %s' % headerval)
             self.ranges.append((start, end))
 
     def __str__(self):
@@ -477,38 +498,68 @@ class Range(object):
             string += ','
         return string.rstrip(',')
 
-    def range_for_length(self, length):
+    def ranges_for_length(self, length):
         """
-        range_for_length is used to determine the correct range of bytes to
-        serve from a body, given body length argument and the Range's ranges.
+        This method is used to return multiple ranges for a given length
+        which should represent the length of the underlying content.
+        The constructor method __init__ made sure that any range in ranges
+        list is syntactically valid. So if length is None or size of the
+        ranges is zero, then the Range header should be ignored which will
+        eventually make the response to be 200.
 
-        A limitation of this method is that it can't handle multiple ranges,
-        for compatibility with webob.  This should be fairly easy to extend.
+        If an empty list is returned by this method, it indicates that there
+        are unsatisfiable ranges found in the Range header, 416 will be
+        returned.
 
-        :param length: length of the response body
+        if a returned list has at least one element, the list indicates that
+        there is at least one range valid and the server should serve the
+        request with a 206 status code.
+
+        The start value of each range represents the starting position in
+        the content, the end value represents the ending position. This
+        method purposely adds 1 to the end number because the spec defines
+        the Range to be inclusive.
+
+        The Range spec can be found at the following link:
+        http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.1
+
+        :param length: length of the underlying content
         """
-        if length is None or not self.ranges or len(self.ranges) != 1:
+        # not syntactically valid ranges, must ignore
+        if length is None or not self.ranges or self.ranges == []:
             return None
-        begin, end = self.ranges[0]
-        if begin is None:
-            if end == 0:
-                return None
-            if end > length:
-                return (0, length)
-            return (length - end, length)
-        if end is None:
-            if begin < length:
-                # If a syntactically valid byte-range-set includes at least one
-                # byte-range-spec whose first-byte-pos is LESS THAN THE CURRENT
-                # LENGTH OF THE ENTITY-BODY..., then the byte-range-set is
-                # satisfiable.
-                return (begin, length)
-            else:
-                # Otherwise, the byte-range-set is unsatisfiable.
-                return None
-        if begin > length:
-            return None
-        return (begin, min(end + 1, length))
+        all_ranges = []
+        for single_range in self.ranges:
+            begin, end = single_range
+            # The possible values for begin and end are
+            # None, 0, or a positive numeric number
+            if begin is None:
+                if end == 0:
+                    # this is the bytes=-0 case
+                    continue
+                elif end > length:
+                    # This is the case where the end is greater than the
+                    # content length, as the RFC 2616 stated, the entire
+                    # content should be returned.
+                    all_ranges.append((0, length))
+                else:
+                    all_ranges.append((length - end, length))
+                continue
+            # begin can only be 0 and numeric value from this point on
+            if end is None:
+                if begin < length:
+                    all_ranges.append((begin, length))
+                else:
+                    # the begin position is greater than or equal to the
+                    # content length; skip and move on to the next range
+                    continue
+            # end can only be 0 or numeric value
+            elif begin < length:
+                # the begin position is valid, take the min of end + 1 or
+                # the total length of the content
+                all_ranges.append((begin, min(end + 1, length)))
+
+        return all_ranges
 
 
 class Match(object):
@@ -759,6 +810,25 @@ class Request(object):
                         app_iter=app_iter, request=self)
 
 
+def content_range_header(start, stop, size, value_only=True):
+    if value_only:
+        range_str = 'bytes %s-%s/%s'
+    else:
+        range_str = 'Content-Range: bytes %s-%s/%s'
+    return range_str % (start, (stop - 1), size)
+
+
+def multi_range_iterator(ranges, content_type, boundary, size, sub_iter_gen):
+    for start, stop in ranges:
+        yield ''.join(['\r\n--', boundary, '\r\n',
+                       'Content-Type: ', content_type, '\r\n'])
+        yield content_range_header(start, stop, size, False) + '\r\n\r\n'
+        sub_iter = sub_iter_gen(start, stop)
+        for chunk in sub_iter:
+            yield chunk
+    yield '\r\n--' + boundary + '--\r\n'
+
+
 class Response(object):
     """
     WSGI Response object.
@@ -783,6 +853,7 @@ class Response(object):
         self.body = body
         self.app_iter = app_iter
         self.status = status
+        self.boundary = "%x" % random.randint(0, 256 ** 16)
         if request:
             self.environ = request.environ
             if request.range and self.status == 200:
@@ -793,6 +864,35 @@ class Response(object):
         for key, value in kw.iteritems():
             setattr(self, key, value)
 
+    def _prepare_for_ranges(self, ranges):
+        """
+        Prepare the Response for multiple ranges.
+        """
+
+        content_size = self.content_length
+        content_type = self.content_type
+        self.content_type = ''.join(['multipart/byteranges;',
+                                     'boundary=', self.boundary])
+
+        # This section calculate the total size of the targeted response
+        # The value 12 is the length of total bytes of hyphen, new line
+        # form feed for each section header. The value 8 is the length of
+        # total bytes of hyphen, new line, form feed characters for the
+        # closing boundary which appears only once
+        section_header_fixed_len = 12 + (len(self.boundary) +
+                                         len('Content-Type: ') +
+                                         len(content_type) +
+                                         len('Content-Range: bytes '))
+        body_size = 0
+        for start, end in ranges:
+            body_size += section_header_fixed_len
+            body_size += len(str(start) + '-' + str(end - 1) + '/' +
+                             str(content_size)) + (end - start)
+        body_size += 8 + len(self.boundary)
+        self.content_length = body_size
+        self.content_range = None
+        return content_size, content_type
+
     def _response_iter(self, app_iter, body):
         if self.request and self.request.method == 'HEAD':
             # We explicitly do NOT want to set self.content_length to 0 here
@@ -800,23 +900,54 @@ class Response(object):
         if self.conditional_response and self.request and \
                 self.request.range and self.request.range.ranges and \
                 not self.content_range:
-            args = self.request.range.range_for_length(self.content_length)
-            if not args:
+            ranges = self.request.range.ranges_for_length(self.content_length)
+            if ranges == []:
                 self.status = 416
                 self.content_length = 0
                 return ['']
-            else:
-                start, end = args
-                self.status = 206
-                self.content_range = self.request.range
-                self.content_length = (end - start)
-                if app_iter and hasattr(app_iter, 'app_iter_range'):
-                    return app_iter.app_iter_range(start, end)
-                elif app_iter:
-                    # this could be improved, but we don't actually use it
-                    return [''.join(app_iter)[start:end]]
-                elif body:
-                    return [body[start:end]]
+            elif ranges:
+                range_size = len(ranges)
+                if range_size > 0:
+                    # There is at least one valid range in the request, so try
+                    # to satisfy the request
+                    if range_size == 1:
+                        start, end = ranges[0]
+                        if app_iter and hasattr(app_iter, 'app_iter_range'):
+                            self.status = 206
+                            self.content_range = \
+                                content_range_header(start, end,
+                                                     self.content_length,
+                                                     True)
+                            self.content_length = (end - start)
+                            return app_iter.app_iter_range(start, end)
+                        elif body:
+                            self.status = 206
+                            self.content_range = \
+                                content_range_header(start, end,
+                                                     self.content_length,
+                                                     True)
+                            self.content_length = (end - start)
+                            return [body[start:end]]
+                    elif range_size > 1:
+                        if app_iter and hasattr(app_iter, 'app_iter_ranges'):
+                            self.status = 206
+                            content_size, content_type = \
+                                self._prepare_for_ranges(ranges)
+                            return app_iter.app_iter_ranges(ranges,
+                                                            content_type,
+                                                            self.boundary,
+                                                            content_size)
+                        elif body:
+                            self.status = 206
+                            content_size, content_type, = \
+                                self._prepare_for_ranges(ranges)
+
+                            def _body_slicer(start, stop):
+                                yield body[start:stop]
+                            return multi_range_iterator(ranges, content_type,
+                                                        self.boundary,
+                                                        content_size,
+                                                        _body_slicer)
         if app_iter:
             return app_iter
         if body:

swift/obj/server.py

@@ -46,7 +46,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
     HTTPInternalServerError, HTTPNoContent, HTTPNotFound, HTTPNotModified, \
     HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
     HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, UTC, \
-    HTTPInsufficientStorage
+    HTTPInsufficientStorage, multi_range_iterator
 
 
 DATADIR = 'objects'
@@ -128,6 +128,7 @@ class DiskFile(object):
         self.read_to_eof = False
         self.quarantined_dir = None
         self.keep_cache = False
+        self.suppress_file_closing = False
         if not os.path.exists(self.datadir):
             return
         files = sorted(os.listdir(self.datadir), reverse=True)
@@ -183,11 +184,12 @@ class DiskFile(object):
                                     read - dropped_cache)
                     break
         finally:
-            self.close()
+            if not self.suppress_file_closing:
+                self.close()
 
     def app_iter_range(self, start, stop):
         """Returns an iterator over the data file for range (start, stop)"""
-        if start:
+        if start or start == 0:
             self.fp.seek(start)
         if stop is not None:
             length = stop - start
@@ -202,6 +204,21 @@ class DiskFile(object):
                     break
             yield chunk
 
+    def app_iter_ranges(self, ranges, content_type, boundary, size):
+        """Returns an iterator over the data file for a set of ranges"""
+        if (not ranges or len(ranges) == 0):
+            yield ''
+        else:
+            try:
+                self.suppress_file_closing = True
+                for chunk in multi_range_iterator(
+                        ranges, content_type, boundary, size,
+                        self.app_iter_range):
+                    yield chunk
+            finally:
+                self.suppress_file_closing = False
+                self.close()
+
     def _handle_close_quarantine(self):
         """Check if file needs to be quarantined"""
         try:

test/unit/common/test_swob.py

@@ -17,6 +17,7 @@
 
 import unittest
 import datetime
+import re
 from StringIO import StringIO
 
 import swift.common.swob
@@ -108,7 +109,7 @@ class TestRange(unittest.TestCase):
 
     def test_upsidedown_range(self):
         range = swift.common.swob.Range('bytes=5-10')
-        self.assertEquals(range.range_for_length(2), None)
+        self.assertEquals(range.ranges_for_length(2), [])
 
     def test_str(self):
         for range_str in ('bytes=1-7', 'bytes=1-', 'bytes=-1',
@@ -116,31 +117,94 @@ class TestRange(unittest.TestCase):
             range = swift.common.swob.Range(range_str)
             self.assertEquals(str(range), range_str)
 
-    def test_range_for_length(self):
+    def test_ranges_for_length(self):
         range = swift.common.swob.Range('bytes=1-7')
-        self.assertEquals(range.range_for_length(10), (1, 8))
-        self.assertEquals(range.range_for_length(5), (1, 5))
-        self.assertEquals(range.range_for_length(None), None)
+        self.assertEquals(range.ranges_for_length(10), [(1, 8)])
+        self.assertEquals(range.ranges_for_length(5), [(1, 5)])
+        self.assertEquals(range.ranges_for_length(None), None)
 
-    def test_range_for_length_no_end(self):
+    def test_ranges_for_large_length(self):
+        range = swift.common.swob.Range('bytes=-1000000000000000000000000000')
+        self.assertEquals(range.ranges_for_length(100), [(0, 100)])
+
+    def test_ranges_for_length_no_end(self):
         range = swift.common.swob.Range('bytes=1-')
-        self.assertEquals(range.range_for_length(10), (1, 10))
-        self.assertEquals(range.range_for_length(5), (1, 5))
-        self.assertEquals(range.range_for_length(None), None)
+        self.assertEquals(range.ranges_for_length(10), [(1, 10)])
+        self.assertEquals(range.ranges_for_length(5), [(1, 5)])
+        self.assertEquals(range.ranges_for_length(None), None)
         # This used to freak out:
         range = swift.common.swob.Range('bytes=100-')
-        self.assertEquals(range.range_for_length(5), None)
-        self.assertEquals(range.range_for_length(None), None)
+        self.assertEquals(range.ranges_for_length(5), [])
+        self.assertEquals(range.ranges_for_length(None), None)
 
-    def test_range_for_length_no_start(self):
+        range = swift.common.swob.Range('bytes=4-6,100-')
+        self.assertEquals(range.ranges_for_length(5), [(4, 5)])
+
+    def test_ranges_for_length_no_start(self):
         range = swift.common.swob.Range('bytes=-7')
-        self.assertEquals(range.range_for_length(10), (3, 10))
-        self.assertEquals(range.range_for_length(5), (0, 5))
-        self.assertEquals(range.range_for_length(None), None)
+        self.assertEquals(range.ranges_for_length(10), [(3, 10)])
+        self.assertEquals(range.ranges_for_length(5), [(0, 5)])
+        self.assertEquals(range.ranges_for_length(None), None)
+
+        range = swift.common.swob.Range('bytes=4-6,-100')
+        self.assertEquals(range.ranges_for_length(5), [(4, 5), (0, 5)])
+
+    def test_ranges_for_length_multi(self):
+        range = swift.common.swob.Range('bytes=-20,4-,30-150,-10')
+        # the length of the ranges should be 4
+        self.assertEquals(len(range.ranges_for_length(200)), 4)
+
+        # the actual length less than any of the range
+        self.assertEquals(range.ranges_for_length(90),
+                          [(70, 90), (4, 90), (30, 90), (80, 90)])
+
+        # the actual length greater than any of the range
+        self.assertEquals(range.ranges_for_length(200),
+                          [(180, 200), (4, 200), (30, 151), (190, 200)])
+
+        self.assertEquals(range.ranges_for_length(None), None)
+
+    def test_ranges_for_length_edges(self):
+        range = swift.common.swob.Range('bytes=0-1, -7')
+        self.assertEquals(range.ranges_for_length(10),
+                          [(0, 2), (3, 10)])
+
+        range = swift.common.swob.Range('bytes=-7, 0-1')
+        self.assertEquals(range.ranges_for_length(10),
+                          [(3, 10), (0, 2)])
+
+        range = swift.common.swob.Range('bytes=-7, 0-1')
+        self.assertEquals(range.ranges_for_length(5),
+                          [(0, 5), (0, 2)])
 
     def test_range_invalid_syntax(self):
-        range = swift.common.swob.Range('bytes=10-2')
-        self.assertEquals(range.ranges, [])
+
+        def _check_invalid_range(range_value):
+            try:
+                swift.common.swob.Range(range_value)
+                return False
+            except ValueError:
+                return True
+
+        """
+        All the following cases should result ValueError exception
+        1. value not starts with bytes=
+        2. range value start is greater than the end, eg. bytes=5-3
+        3. range does not have start or end, eg. bytes=-
+        4. range does not have hyphen, eg. bytes=45
+        5. range value is non numeric
+        6. any combination of the above
+        """
+
+        self.assert_(_check_invalid_range('nonbytes=foobar,10-2'))
+        self.assert_(_check_invalid_range('bytes=5-3'))
+        self.assert_(_check_invalid_range('bytes=-'))
+        self.assert_(_check_invalid_range('bytes=45'))
+        self.assert_(_check_invalid_range('bytes=foo-bar,3-5'))
+        self.assert_(_check_invalid_range('bytes=4-10,45'))
+        self.assert_(_check_invalid_range('bytes=foobar,3-5'))
+        self.assert_(_check_invalid_range('bytes=nonumber-5'))
+        self.assert_(_check_invalid_range('bytes=nonumber'))
 
 
 class TestMatch(unittest.TestCase):
@@ -387,6 +451,106 @@ class TestResponse(unittest.TestCase):
         body = ''.join(resp({}, start_response))
         self.assertEquals(body, 'abc')
 
+    def test_multi_ranges_wo_iter_ranges(self):
+        def test_app(environ, start_response):
+            start_response('200 OK', [('Content-Length', '10')])
+            return ['1234567890']
+
+        req = swift.common.swob.Request.blank(
+            '/', headers={'Range': 'bytes=0-9,10-19,20-29'})
+
+        resp = req.get_response(test_app)
+        resp.conditional_response = True
+        resp.content_length = 10
+
+        content = ''.join(resp._response_iter(resp.app_iter, ''))
+
+        self.assertEquals(resp.status, '200 OK')
+        self.assertEqual(10, resp.content_length)
+
+    def test_single_range_wo_iter_range(self):
+        def test_app(environ, start_response):
+            start_response('200 OK', [('Content-Length', '10')])
+            return ['1234567890']
+
+        req = swift.common.swob.Request.blank(
+            '/', headers={'Range': 'bytes=0-9'})
+
+        resp = req.get_response(test_app)
+        resp.conditional_response = True
+        resp.content_length = 10
+
+        content = ''.join(resp._response_iter(resp.app_iter, ''))
+
+        self.assertEquals(resp.status, '200 OK')
+        self.assertEqual(10, resp.content_length)
+
+    def test_multi_range_body(self):
+        def test_app(environ, start_response):
+            start_response('200 OK', [('Content-Length', '4')])
+            return ['abcd']
+
+        req = swift.common.swob.Request.blank(
+            '/', headers={'Range': 'bytes=0-9,10-19,20-29'})
+
+        resp = req.get_response(test_app)
+        resp.conditional_response = True
+        resp.content_length = 100
+
+        resp.content_type = 'text/plain'
+        content = ''.join(resp._response_iter(None,
+                                              ('0123456789112345678'
+                                               '92123456789')))
+
+        self.assert_(re.match(('\r\n'
+                               '--[a-f0-9]{32}\r\n'
+                               'Content-Type: text/plain\r\n'
+                               'Content-Range: bytes '
+                               '0-9/100\r\n\r\n0123456789\r\n'
+                               '--[a-f0-9]{32}\r\n'
+                               'Content-Type: text/plain\r\n'
+                               'Content-Range: bytes '
+                               '10-19/100\r\n\r\n1123456789\r\n'
+                               '--[a-f0-9]{32}\r\n'
+                               'Content-Type: text/plain\r\n'
+                               'Content-Range: bytes '
+                               '20-29/100\r\n\r\n2123456789\r\n'
+                               '--[a-f0-9]{32}--\r\n'), content))
+
+    def test_multi_response_iter(self):
+        def test_app(environ, start_response):
+            start_response('200 OK', [('Content-Length', '10'),
+                                      ('Content-Type', 'application/xml')])
+            return ['0123456789']
+
+        app_iter_ranges_args = []
+
+        class App_iter(object):
+            def app_iter_ranges(self, ranges, content_type, boundary, size):
+                app_iter_ranges_args.append((ranges, content_type, boundary,
+                                             size))
+                for i in xrange(3):
+                    yield str(i) + 'fun'
+                yield boundary
+
+            def __iter__(self):
+                for i in xrange(3):
+                    yield str(i) + 'fun'
+
+        req = swift.common.swob.Request.blank(
+            '/', headers={'Range': 'bytes=1-5,8-11'})
+
+        resp = req.get_response(test_app)
+        resp.conditional_response = True
+        resp.content_length = 12
+
+        content = ''.join(resp._response_iter(App_iter(), ''))
+        boundary = content[-32:]
+        self.assertEqual(content[:-32], '0fun1fun2fun')
+        self.assertEqual(app_iter_ranges_args,
+                         [([(1, 6), (8, 12)], 'application/xml',
+                           boundary, 12)])
+
     def test_range_body(self):
 
         def test_app(environ, start_response):
@@ -398,20 +562,17 @@ class TestResponse(unittest.TestCase):
 
         req = swift.common.swob.Request.blank(
             '/', headers={'Range': 'bytes=1-3'})
-        resp = req.get_response(test_app)
-        resp.conditional_response = True
-        body = ''.join(resp([], start_response))
-        self.assertEquals(body, '234')
-        self.assertEquals(resp.status, '206 Partial Content')
 
         resp = swift.common.swob.Response(
             body='1234567890', request=req,
             conditional_response=True)
         body = ''.join(resp([], start_response))
         self.assertEquals(body, '234')
+        self.assertEquals(resp.content_range, 'bytes 1-3/10')
         self.assertEquals(resp.status, '206 Partial Content')
 
-        # No body for 416
+        # syntactically valid, but does not make sense, so returning 416
+        # in next couple of cases.
         req = swift.common.swob.Request.blank(
             '/', headers={'Range': 'bytes=-0'})
         resp = req.get_response(test_app)
@@ -426,6 +587,7 @@ class TestResponse(unittest.TestCase):
             conditional_response=True)
         body = ''.join(resp([], start_response))
         self.assertEquals(body, '')
+        self.assertEquals(resp.content_length, 0)
         self.assertEquals(resp.status, '416 Requested Range Not Satisfiable')
 
         # Syntactically-invalid Range headers "MUST" be ignored

test/unit/obj/test_server.py

@@ -18,6 +18,7 @@
 import cPickle as pickle
 import os
 import unittest
+import email
 from shutil import rmtree
 from StringIO import StringIO
 from time import gmtime, sleep, strftime, time
@@ -55,31 +56,89 @@ class TestDiskFile(unittest.TestCase):
         """ Tear down for testing swift.object_server.ObjectController """
         rmtree(os.path.dirname(self.testdir))
 
-    def test_disk_file_app_iter_corners(self):
+    def _create_test_file(self, data, keep_data_fp=True):
         df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
                                     FakeLogger())
         mkdirs(df.datadir)
         f = open(os.path.join(df.datadir,
                               normalize_timestamp(time()) + '.data'), 'wb')
-        f.write('1234567890')
+        f.write(data)
         setxattr(f.fileno(), object_server.METADATA_KEY,
                  pickle.dumps({}, object_server.PICKLE_PROTOCOL))
         f.close()
         df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
-                                    FakeLogger(), keep_data_fp=True)
-        it = df.app_iter_range(0, None)
-        sio = StringIO()
-        for chunk in it:
-            sio.write(chunk)
-        self.assertEquals(sio.getvalue(), '1234567890')
+                                    FakeLogger(), keep_data_fp=keep_data_fp)
+        return df
+
+    def test_disk_file_app_iter_corners(self):
+        df = self._create_test_file('1234567890')
+        self.assertEquals(''.join(df.app_iter_range(0, None)), '1234567890')
 
         df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
                                     FakeLogger(), keep_data_fp=True)
-        it = df.app_iter_range(5, None)
-        sio = StringIO()
-        for chunk in it:
-            sio.write(chunk)
-        self.assertEquals(sio.getvalue(), '67890')
+        self.assertEqual(''.join(df.app_iter_range(5, None)), '67890')
+
+    def test_disk_file_app_iter_ranges(self):
+        df = self._create_test_file('012345678911234567892123456789')
+        it = df.app_iter_ranges([(0, 10), (10, 20), (20, 30)], 'plain/text',
+                                '\r\n--someheader\r\n', 30)
+        value = ''.join(it)
+        self.assert_('0123456789' in value)
+        self.assert_('1123456789' in value)
+        self.assert_('2123456789' in value)
+
+    def test_disk_file_app_iter_ranges_edges(self):
+        df = self._create_test_file('012345678911234567892123456789')
+        it = df.app_iter_ranges([(3, 10), (0, 2)], 'application/whatever',
+                                '\r\n--someheader\r\n', 30)
+        value = ''.join(it)
+        self.assert_('3456789' in value)
+        self.assert_('01' in value)
+
+    def test_disk_file_large_app_iter_ranges(self):
+        """
+        This test case is to make sure that the disk file app_iter_ranges
+        method all the paths being tested.
+        """
+        long_str = '01234567890' * 65536
+        target_strs = ['3456789', long_str[0:65590]]
+        df = self._create_test_file(long_str)
+
+        it = df.app_iter_ranges([(3, 10), (0, 65590)], 'plain/text',
+                                '5e816ff8b8b8e9a5d355497e5d9e0301', 655360)
+
+        """
+        the produced string actually missing the MIME headers
+        need to add these headers to make it as real MIME message.
+        The body of the message is produced by method app_iter_ranges
+        off of DiskFile object.
+        """
+        header = ''.join(['Content-Type: multipart/byteranges;',
+                          'boundary=',
+                          '5e816ff8b8b8e9a5d355497e5d9e0301\r\n'])
+
+        value = header + ''.join(it)
+
+        parts = map(lambda p: p.get_payload(decode=True),
+                    email.message_from_string(value).walk())[1:3]
+        self.assertEqual(parts, target_strs)
+
+    def test_disk_file_app_iter_ranges_empty(self):
+        """
+        This test case tests when empty value passed into app_iter_ranges
+        When ranges passed into the method is either empty array or None,
+        this method will yield empty string
+        """
+        df = self._create_test_file('012345678911234567892123456789')
+        it = df.app_iter_ranges([], 'application/whatever',
+                                '\r\n--someheader\r\n', 100)
+        self.assertEqual(''.join(it), '')
+
+        df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
+                                    FakeLogger(), keep_data_fp=True)
+        it = df.app_iter_ranges(None, 'app/something',
+                                '\r\n--someheader\r\n', 150)
+        self.assertEqual(''.join(it), '')
 
     def test_disk_file_mkstemp_creates_dir(self):
         tmpdir = os.path.join(self.testdir, 'sda1', 'tmp')

test/unit/proxy/test_server.py

@@ -4450,9 +4450,9 @@ class FakeObjectController(object):
         path = args[4]
         body = data = path[-1] * int(path[-1])
         if req.range:
-            r = req.range.range_for_length(len(data))
+            r = req.range.ranges_for_length(len(data))
             if r:
-                (start, stop) = r
+                (start, stop) = r[0]
                 body = data[start:stop]
         resp = Response(app_iter=iter(body))
         return resp