swift/test/unit/proxy/test_server.py

# Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import with_statement
import cPickle as pickle
import logging
import os
import sys
import unittest
import urlparse
import signal
from contextlib import contextmanager, nested
from gzip import GzipFile
from shutil import rmtree
import time
from urllib import quote
from hashlib import md5
from tempfile import mkdtemp
import random

import mock
from eventlet import sleep, spawn, wsgi, listen
import simplejson

from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, fake_http_connect
from swift.proxy import server as proxy_server
from swift.account import server as account_server
from swift.container import server as container_server
from swift.obj import server as object_server
from swift.common import ring
from swift.common.exceptions import ChunkReadTimeout
from swift.common.constraints import MAX_META_NAME_LENGTH, \
    MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \
    MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH
from swift.common import utils
from swift.common.utils import mkdirs, normalize_timestamp, NullLogger
from swift.common.wsgi import monkey_patch_mimetools
from swift.proxy.controllers.obj import SegmentedIterable
from swift.proxy.controllers.base import get_container_memcache_key, \
    get_account_memcache_key, cors_validation
import swift.proxy.controllers
from swift.common.swob import Request, Response, HTTPNotFound, \
    HTTPUnauthorized

# mocks
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))


_request_instances = 0


def request_init(self, *args, **kwargs):
    global _request_instances
    self._orig_init(*args, **kwargs)
    _request_instances += 1


def request_del(self):
    global _request_instances
    if self._orig_del:
        self._orig_del()
    _request_instances -= 1


def setup():
    utils.HASH_PATH_SUFFIX = 'endcap'
    global _testdir, _test_servers, _test_sockets, \
        _orig_container_listing_limit, _test_coros
    Request._orig_init = Request.__init__
    Request.__init__ = request_init
    Request._orig_del = getattr(Request, '__del__', None)
    Request.__del__ = request_del
    monkey_patch_mimetools()
    # Since we're starting up a lot here, we're going to test more than
    # just chunked puts; we're also going to test parts of
    # proxy_server.Application we couldn't get to easily otherwise.
    _testdir = \
        os.path.join(mkdtemp(), 'tmp_test_proxy_server_chunked')
    mkdirs(_testdir)
    rmtree(_testdir)
    mkdirs(os.path.join(_testdir, 'sda1'))
    mkdirs(os.path.join(_testdir, 'sda1', 'tmp'))
    mkdirs(os.path.join(_testdir, 'sdb1'))
    mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))
    _orig_container_listing_limit = \
        swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT
    conf = {'devices': _testdir, 'swift_dir': _testdir,
            'mount_check': 'false', 'allowed_headers':
            'content-encoding, x-object-manifest, content-disposition, foo',
            'allow_versions': 'True'}
    prolis = listen(('localhost', 0))
    acc1lis = listen(('localhost', 0))
    acc2lis = listen(('localhost', 0))
    con1lis = listen(('localhost', 0))
    con2lis = listen(('localhost', 0))
    obj1lis = listen(('localhost', 0))
    obj2lis = listen(('localhost', 0))
    _test_sockets = \
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis)
    pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
                  'port': acc1lis.getsockname()[1]},
                 {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
                  'port': acc2lis.getsockname()[1]}], 30),
                GzipFile(os.path.join(_testdir, 'account.ring.gz'), 'wb'))
    pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
                  'port': con1lis.getsockname()[1]},
                 {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
                  'port': con2lis.getsockname()[1]}], 30),
                GzipFile(os.path.join(_testdir, 'container.ring.gz'), 'wb'))
    pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
                  'port': obj1lis.getsockname()[1]},
                 {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
                  'port': obj2lis.getsockname()[1]}], 30),
                GzipFile(os.path.join(_testdir, 'object.ring.gz'), 'wb'))
    prosrv = proxy_server.Application(conf, FakeMemcacheReturnsNone())
    acc1srv = account_server.AccountController(conf)
    acc2srv = account_server.AccountController(conf)
    con1srv = container_server.ContainerController(conf)
    con2srv = container_server.ContainerController(conf)
    obj1srv = object_server.ObjectController(conf)
    obj2srv = object_server.ObjectController(conf)
    _test_servers = \
        (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv)
    nl = NullLogger()
    prospa = spawn(wsgi.server, prolis, prosrv, nl)
    acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl)
    acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl)
    con1spa = spawn(wsgi.server, con1lis, con1srv, nl)
    con2spa = spawn(wsgi.server, con2lis, con2srv, nl)
    obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl)
    obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl)
    _test_coros = \
        (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa)
    # Create account
    ts = normalize_timestamp(time.time())
    partition, nodes = prosrv.account_ring.get_nodes('a')
    for node in nodes:
        conn = swift.proxy.controllers.obj.http_connect(node['ip'],
                                                        node['port'],
                                                        node['device'],
                                                        partition, 'PUT', '/a',
                                                        {'X-Timestamp': ts,
                                                         'x-trans-id': 'test'})
        resp = conn.getresponse()
        assert(resp.status == 201)
    # Create container
    sock = connect_tcp(('localhost', prolis.getsockname()[1]))
    fd = sock.makefile()
    fd.write('PUT /v1/a/c HTTP/1.1\r\nHost: localhost\r\n'
             'Connection: close\r\nX-Auth-Token: t\r\n'
             'Content-Length: 0\r\n\r\n')
    fd.flush()
    headers = readuntil2crlfs(fd)
    exp = 'HTTP/1.1 201'
    assert(headers[:len(exp)] == exp)


def teardown():
    for server in _test_coros:
        server.kill()
    swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = \
        _orig_container_listing_limit
    rmtree(os.path.dirname(_testdir))
    Request.__init__ = Request._orig_init
    if Request._orig_del:
        Request.__del__ = Request._orig_del


def sortHeaderNames(headerNames):
    """
    Return the given string of header names sorted.

    headerName: a comma-delimited list of header names
    """
    headers = [a.strip() for a in headerNames.split(',') if a.strip()]
    headers.sort()
    return ', '.join(headers)


class FakeRing(object):

    def __init__(self, replicas=3):
        # 9 total nodes (6 more past the initial 3) is the cap, no matter if
        # this is set higher, or R^2 for R replicas
        self.replicas = replicas
        self.max_more_nodes = 0
        self.devs = {}

    def set_replicas(self, replicas):
        self.replicas = replicas
        self.devs = {}

    @property
    def replica_count(self):
        return self.replicas

    def get_part(self, account, container=None, obj=None):
        return 1

    def get_nodes(self, account, container=None, obj=None):
        devs = []
        for x in xrange(self.replicas):
            devs.append(self.devs.get(x))
            if devs[x] is None:
                self.devs[x] = devs[x] = \
                    {'ip': '10.0.0.%s' % x,
                     'port': 1000 + x,
                     'device': 'sd' + (chr(ord('a') + x)),
                     'id': x}
        return 1, devs

    def get_part_nodes(self, part):
        return self.get_nodes('blah')[1]

    def get_more_nodes(self, nodes):
        # replicas^2 is the true cap
        for x in xrange(self.replicas, min(self.replicas + self.max_more_nodes,
                                           self.replicas * self.replicas)):
            yield {'ip': '10.0.0.%s' % x, 'port': 1000 + x, 'device': 'sda'}


class FakeMemcache(object):

    def __init__(self):
        self.store = {}

    def get(self, key):
        return self.store.get(key)

    def keys(self):
        return self.store.keys()

    def set(self, key, value, time=0):
        self.store[key] = value
        return True

    def incr(self, key, time=0):
        self.store[key] = self.store.setdefault(key, 0) + 1
        return self.store[key]

    @contextmanager
    def soft_lock(self, key, timeout=0, retries=5):
        yield True

    def delete(self, key):
        try:
            del self.store[key]
        except Exception:
            pass
        return True


class FakeMemcacheReturnsNone(FakeMemcache):

    def get(self, key):
        # Returns None as the timestamp of the container; assumes we're only
        # using the FakeMemcache for container existence checks.
        return None


@contextmanager
def save_globals():
    orig_http_connect = getattr(swift.proxy.controllers.base, 'http_connect',
                                None)
    orig_account_info = getattr(swift.proxy.controllers.Controller,
                                'account_info', None)
    try:
        yield True
    finally:
        swift.proxy.controllers.Controller.account_info = orig_account_info
        swift.proxy.controllers.base.http_connect = orig_http_connect
        swift.proxy.controllers.obj.http_connect = orig_http_connect
        swift.proxy.controllers.account.http_connect = orig_http_connect
        swift.proxy.controllers.container.http_connect = orig_http_connect


def set_http_connect(*args, **kwargs):
    new_connect = fake_http_connect(*args, **kwargs)
    swift.proxy.controllers.base.http_connect = new_connect
    swift.proxy.controllers.obj.http_connect = new_connect
    swift.proxy.controllers.account.http_connect = new_connect
    swift.proxy.controllers.container.http_connect = new_connect


# tests
class TestController(unittest.TestCase):

    def setUp(self):
        self.account_ring = FakeRing()
        self.container_ring = FakeRing()
        self.memcache = FakeMemcache()

        app = proxy_server.Application(None, self.memcache,
                                       account_ring=self.account_ring,
                                       container_ring=self.container_ring,
                                       object_ring=FakeRing())
        self.controller = swift.proxy.controllers.Controller(app)

        self.account = 'some_account'
        self.container = 'some_container'
        self.read_acl = 'read_acl'
        self.write_acl = 'write_acl'

    def test_transfer_headers(self):
        src_headers = {'x-remove-base-meta-owner': 'x',
                       'x-base-meta-size': '151M',
                       'new-owner': 'Kun'}
        dst_headers = {'x-base-meta-owner': 'Gareth',
                       'x-base-meta-size': '150M'}
        self.controller.transfer_headers(src_headers, dst_headers)
        expected_headers = {'x-base-meta-owner': '',
                            'x-base-meta-size': '151M'}
        self.assertEquals(dst_headers, expected_headers)

    def check_account_info_return(self, partition, nodes, is_none=False):
        if is_none:
            p, n = None, None
        else:
            p, n = self.account_ring.get_nodes(self.account)
        self.assertEqual(p, partition)
        self.assertEqual(n, nodes)

    def test_account_info_container_count(self):
        with save_globals():
            set_http_connect(200, count=123)
            partition, nodes, count = \
                self.controller.account_info(self.account)
            self.assertEquals(count, 123)
        with save_globals():
            set_http_connect(200, count='123')
            partition, nodes, count = \
                self.controller.account_info(self.account)
            self.assertEquals(count, 123)
        with save_globals():
            cache_key = get_account_memcache_key(self.account)
            account_info = {'status': 200, 'container_count': 1234}
            self.memcache.set(cache_key, account_info)
            partition, nodes, count = \
                self.controller.account_info(self.account)
            self.assertEquals(count, 1234)
        with save_globals():
            cache_key = get_account_memcache_key(self.account)
            account_info = {'status': 200, 'container_count': '1234'}
            self.memcache.set(cache_key, account_info)
            partition, nodes, count = \
                self.controller.account_info(self.account)
            self.assertEquals(count, 1234)

    def test_make_requests(self):
        with save_globals():
            set_http_connect(200)
            partition, nodes, count = \
                self.controller.account_info(self.account)
            set_http_connect(201, raise_timeout_exc=True)
            self.controller._make_request(
                nodes, partition, 'POST', '/', '', '',
                self.controller.app.logger.thread_locals)

    # tests if 200 is cached and used
    def test_account_info_200(self):
        with save_globals():
            set_http_connect(200)
            partition, nodes, count = \
                self.controller.account_info(self.account)
            self.check_account_info_return(partition, nodes)
            self.assertEquals(count, 12345)

            cache_key = get_account_memcache_key(self.account)
            container_info = {'status': 200,
                              'container_count': 12345,
                              'total_object_count': None,
                              'bytes': None,
                              'meta': {}}
            self.assertEquals(container_info,
                              self.memcache.get(cache_key))

            set_http_connect()
            partition, nodes, count = \
                self.controller.account_info(self.account)
            self.check_account_info_return(partition, nodes)
            self.assertEquals(count, 12345)

    # tests if 404 is cached and used
    def test_account_info_404(self):
        with save_globals():
            set_http_connect(404, 404, 404)
            partition, nodes, count = \
                self.controller.account_info(self.account)
            self.check_account_info_return(partition, nodes, True)
            self.assertEquals(count, None)

            cache_key = get_account_memcache_key(self.account)
            container_info = {'status': 404,
                              'container_count': 0,
                              'total_object_count': None,
                              'bytes': None,
                              'meta': {}}
            self.assertEquals(container_info,
                              self.memcache.get(cache_key))

            set_http_connect()
            partition, nodes, count = \
                self.controller.account_info(self.account)
            self.check_account_info_return(partition, nodes, True)
            self.assertEquals(count, None)

    # tests if some http status codes are not cached
    def test_account_info_no_cache(self):
        def test(*status_list):
            set_http_connect(*status_list)
            partition, nodes, count = \
                self.controller.account_info(self.account)
            self.assertEqual(len(self.memcache.keys()), 0)
            self.check_account_info_return(partition, nodes, True)
            self.assertEquals(count, None)

        with save_globals():
            test(503, 404, 404)
            test(404, 404, 503)
            test(404, 507, 503)
            test(503, 503, 503)

    def test_account_info_account_autocreate(self):
        with save_globals():
            self.memcache.store = {}
            set_http_connect(404, 404, 404, 201, 201, 201)
            partition, nodes, count = \
                self.controller.account_info(self.account, autocreate=False)
            self.check_account_info_return(partition, nodes, is_none=True)
            self.assertEquals(count, None)

            self.memcache.store = {}
            set_http_connect(404, 404, 404, 201, 201, 201)
            partition, nodes, count = \
                self.controller.account_info(self.account)
            self.check_account_info_return(partition, nodes, is_none=True)
            self.assertEquals(count, None)

            self.memcache.store = {}
            set_http_connect(404, 404, 404, 201, 201, 201)
            partition, nodes, count = \
                self.controller.account_info(self.account, autocreate=True)
            self.check_account_info_return(partition, nodes)
            self.assertEquals(count, 0)

            self.memcache.store = {}
            set_http_connect(404, 404, 404, 503, 201, 201)
            partition, nodes, count = \
                self.controller.account_info(self.account, autocreate=True)
            self.check_account_info_return(partition, nodes)
            self.assertEquals(count, 0)

            self.memcache.store = {}
            set_http_connect(404, 404, 404, 503, 201, 503)
            exc = None
            partition, nodes, count = \
                self.controller.account_info(self.account, autocreate=True)
            self.check_account_info_return(partition, nodes, is_none=True)
            self.assertEquals(None, count)

            self.memcache.store = {}
            set_http_connect(404, 404, 404, 403, 403, 403)
            exc = None
            partition, nodes, count = \
                self.controller.account_info(self.account, autocreate=True)
            self.check_account_info_return(partition, nodes, is_none=True)
            self.assertEquals(None, count)

            self.memcache.store = {}
            set_http_connect(404, 404, 404, 409, 409, 409)
            exc = None
            partition, nodes, count = \
                self.controller.account_info(self.account, autocreate=True)
            self.check_account_info_return(partition, nodes, is_none=True)
            self.assertEquals(None, count)

    def check_container_info_return(self, ret, is_none=False):
        if is_none:
            partition, nodes, read_acl, write_acl = None, None, None, None
        else:
            partition, nodes = self.container_ring.get_nodes(self.account,
                                                             self.container)
            read_acl, write_acl = self.read_acl, self.write_acl
        self.assertEqual(partition, ret['partition'])
        self.assertEqual(nodes, ret['nodes'])
        self.assertEqual(read_acl, ret['read_acl'])
        self.assertEqual(write_acl, ret['write_acl'])

    def test_container_info_invalid_account(self):
        def account_info(self, account, autocreate=False):
            return None, None

        with save_globals():
            swift.proxy.controllers.Controller.account_info = account_info
            ret = self.controller.container_info(self.account,
                                                 self.container)
            self.check_container_info_return(ret, True)

    # tests if 200 is cached and used
    def test_container_info_200(self):
        def account_info(self, account, autocreate=False):
            return True, True, 0

        with save_globals():
            headers = {'x-container-read': self.read_acl,
                       'x-container-write': self.write_acl}
            swift.proxy.controllers.Controller.account_info = account_info
            set_http_connect(200, headers=headers)
            ret = self.controller.container_info(self.account,
                                                 self.container)
            self.check_container_info_return(ret)

            cache_key = get_container_memcache_key(self.account,
                                                   self.container)
            cache_value = self.memcache.get(cache_key)
            self.assertTrue(isinstance(cache_value, dict))
            self.assertEquals(200, cache_value.get('status'))

            set_http_connect()
            ret = self.controller.container_info(self.account,
                                                 self.container)
            self.check_container_info_return(ret)

    # tests if 404 is cached and used
    def test_container_info_404(self):
        def account_info(self, account, autocreate=False):
            return True, True, 0

        with save_globals():
            swift.proxy.controllers.Controller.account_info = account_info
            set_http_connect(404, 404, 404)
            ret = self.controller.container_info(self.account,
                                                 self.container)
            self.check_container_info_return(ret, True)

            cache_key = get_container_memcache_key(self.account,
                                                   self.container)
            cache_value = self.memcache.get(cache_key)
            self.assertTrue(isinstance(cache_value, dict))
            self.assertEquals(404, cache_value.get('status'))

            set_http_connect()
            ret = self.controller.container_info(self.account,
                                                 self.container)
            self.check_container_info_return(ret, True)

    # tests if some http status codes are not cached
    def test_container_info_no_cache(self):
        def test(*status_list):
            set_http_connect(*status_list)
            ret = self.controller.container_info(self.account,
                                                 self.container)
            self.assertEqual(len(self.memcache.keys()), 0)
            self.check_container_info_return(ret, True)

        with save_globals():
            test(503, 404, 404)
            test(404, 404, 503)
            test(404, 507, 503)
            test(503, 503, 503)


class TestProxyServer(unittest.TestCase):

    def test_unhandled_exception(self):

        class MyApp(proxy_server.Application):

            def get_controller(self, path):
                raise Exception('this shouldnt be caught')

        app = MyApp(None, FakeMemcache(), account_ring=FakeRing(),
                    container_ring=FakeRing(), object_ring=FakeRing())
        req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'})
        app.update_request(req)
        resp = app.handle_request(req)
        self.assertEquals(resp.status_int, 500)

    def test_internal_method_request(self):
        baseapp = proxy_server.Application({},
                                           FakeMemcache(),
                                           container_ring=FakeRing(),
                                           object_ring=FakeRing(),
                                           account_ring=FakeRing())
        resp = baseapp.handle_request(
            Request.blank('/v1/a', environ={'REQUEST_METHOD': '__init__'}))
        self.assertEquals(resp.status, '405 Method Not Allowed')

    def test_inexistent_method_request(self):
        baseapp = proxy_server.Application({},
                                           FakeMemcache(),
                                           container_ring=FakeRing(),
                                           account_ring=FakeRing(),
                                           object_ring=FakeRing())
        resp = baseapp.handle_request(
            Request.blank('/v1/a', environ={'REQUEST_METHOD': '!invalid'}))
        self.assertEquals(resp.status, '405 Method Not Allowed')

    def test_calls_authorize_allow(self):
        called = [False]

        def authorize(req):
            called[0] = True
        with save_globals():
            set_http_connect(200)
            app = proxy_server.Application(None, FakeMemcache(),
                                           account_ring=FakeRing(),
                                           container_ring=FakeRing(),
                                           object_ring=FakeRing())
            req = Request.blank('/v1/a')
            req.environ['swift.authorize'] = authorize
            app.update_request(req)
            resp = app.handle_request(req)
        self.assert_(called[0])

    def test_calls_authorize_deny(self):
        called = [False]

        def authorize(req):
            called[0] = True
            return HTTPUnauthorized(request=req)
        app = proxy_server.Application(None, FakeMemcache(),
                                       account_ring=FakeRing(),
                                       container_ring=FakeRing(),
                                       object_ring=FakeRing())
        req = Request.blank('/v1/a')
        req.environ['swift.authorize'] = authorize
        app.update_request(req)
        resp = app.handle_request(req)
        self.assert_(called[0])

    def test_negative_content_length(self):
        swift_dir = mkdtemp()
        try:
            baseapp = proxy_server.Application({'swift_dir': swift_dir},
                                               FakeMemcache(), FakeLogger(),
                                               FakeRing(), FakeRing(),
                                               FakeRing())
            resp = baseapp.handle_request(
                Request.blank('/', environ={'CONTENT_LENGTH': '-1'}))
            self.assertEquals(resp.status, '400 Bad Request')
            self.assertEquals(resp.body, 'Invalid Content-Length')
            resp = baseapp.handle_request(
                Request.blank('/', environ={'CONTENT_LENGTH': '-123'}))
            self.assertEquals(resp.status, '400 Bad Request')
            self.assertEquals(resp.body, 'Invalid Content-Length')
        finally:
            rmtree(swift_dir, ignore_errors=True)

    def test_denied_host_header(self):
        swift_dir = mkdtemp()
        try:
            baseapp = proxy_server.Application({'swift_dir': swift_dir,
                                                'deny_host_headers':
                                                'invalid_host.com'},
                                               FakeMemcache(), FakeLogger(),
                                               FakeRing(), FakeRing(),
                                               FakeRing())
            resp = baseapp.handle_request(
                Request.blank('/v1/a/c/o',
                              environ={'HTTP_HOST': 'invalid_host.com'}))
            self.assertEquals(resp.status, '403 Forbidden')
        finally:
            rmtree(swift_dir, ignore_errors=True)

    def test_node_timing(self):
        baseapp = proxy_server.Application({'sorting_method': 'timing'},
                                           FakeMemcache(),
                                           container_ring=FakeRing(),
                                           object_ring=FakeRing(),
                                           account_ring=FakeRing())
        self.assertEquals(baseapp.node_timings, {})

        req = Request.blank('/v1/account', environ={'REQUEST_METHOD': 'HEAD'})
        baseapp.update_request(req)
        resp = baseapp.handle_request(req)
        self.assertEquals(resp.status_int, 503)  # couldn't connect to anything
        exp_timings = {}
        self.assertEquals(baseapp.node_timings, exp_timings)

        proxy_server.time = lambda: times.pop(0)
        try:
            times = [time.time()]
            exp_timings = {'127.0.0.1': (0.1,
                                         times[0] + baseapp.timing_expiry)}
            baseapp.set_node_timing({'ip': '127.0.0.1'}, 0.1)
            self.assertEquals(baseapp.node_timings, exp_timings)
        finally:
            proxy_server.time = time.time

        proxy_server.shuffle = lambda l: l
        try:
            nodes = [{'ip': '127.0.0.1'}, {'ip': '127.0.0.2'}, {'ip': '127.0.0.3'}]
            res = baseapp.sort_nodes(nodes)
            exp_sorting = [{'ip': '127.0.0.2'}, {'ip': '127.0.0.3'},
                           {'ip': '127.0.0.1'}]
            self.assertEquals(res, exp_sorting)
        finally:
            proxy_server.shuffle = random.shuffle


class TestObjectController(unittest.TestCase):

    def setUp(self):
        self.app = proxy_server.Application(None, FakeMemcache(),
                                            account_ring=FakeRing(),
                                            container_ring=FakeRing(),
                                            object_ring=FakeRing())
        monkey_patch_mimetools()

    def tearDown(self):
        self.app.account_ring.set_replicas(3)
        self.app.container_ring.set_replicas(3)
        self.app.object_ring.set_replicas(3)

    def assert_status_map(self, method, statuses, expected, raise_exc=False):
        with save_globals():
            kwargs = {}
            if raise_exc:
                kwargs['raise_exc'] = raise_exc

            set_http_connect(*statuses, **kwargs)
            self.app.memcache.store = {}
            req = Request.blank('/a/c/o',
                                headers={'Content-Length': '0',
                                         'Content-Type': 'text/plain'})
            self.app.update_request(req)
            res = method(req)
            self.assertEquals(res.status_int, expected)

            # repeat test
            set_http_connect(*statuses, **kwargs)
            self.app.memcache.store = {}
            req = Request.blank('/a/c/o',
                                headers={'Content-Length': '0',
                                         'Content-Type': 'text/plain'})
            self.app.update_request(req)
            res = method(req)
            self.assertEquals(res.status_int, expected)

    def test_GET_newest_large_file(self):
        calls = [0]

        def handler(_junk1, _junk2):
            calls[0] += 1

        old_handler = signal.signal(signal.SIGPIPE, handler)
        try:
            prolis = _test_sockets[0]
            prosrv = _test_servers[0]
            sock = connect_tcp(('localhost', prolis.getsockname()[1]))
            fd = sock.makefile()
            obj = 'a' * (1024 * 1024)
            path = '/v1/a/c/o.large'
            fd.write('PUT %s HTTP/1.1\r\n'
                     'Host: localhost\r\n'
                     'Connection: close\r\n'
                     'X-Storage-Token: t\r\n'
                     'Content-Length: %s\r\n'
                     'Content-Type: application/octet-stream\r\n'
                     '\r\n%s' % (path, str(len(obj)), obj))
            fd.flush()
            headers = readuntil2crlfs(fd)
            exp = 'HTTP/1.1 201'
            self.assertEqual(headers[:len(exp)], exp)
            req = Request.blank(path,
                                environ={'REQUEST_METHOD': 'GET'},
                                headers={'Content-Type':
                                         'application/octet-stream',
                                         'X-Newest': 'true'})
            res = req.get_response(prosrv)
            self.assertEqual(res.status_int, 200)
            self.assertEqual(res.body, obj)
            self.assertEqual(calls[0], 0)
        finally:
            signal.signal(signal.SIGPIPE, old_handler)

    def test_PUT_expect_header_zero_content_length(self):
        test_errors = []

        def test_connect(ipaddr, port, device, partition, method, path,
                         headers=None, query_string=None):
            if path == '/a/c/o.jpg':
                if 'expect' in headers or 'Expect' in headers:
                    test_errors.append('Expect was in headers for object '
                                       'server!')

        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            # The (201, -4) tuples in there have the effect of letting the
            # initial connect succeed, after which getexpect() gets called and
            # then the -4 makes the response of that actually be 201 instead of
            # 100.  Perfectly straightforward.
            set_http_connect(200, 200, (201, -4), (201, -4), (201, -4),
                             give_connect=test_connect)
            req = Request.blank('/a/c/o.jpg', {})
            req.content_length = 0
            self.app.update_request(req)
            self.app.memcache.store = {}
            res = controller.PUT(req)
            self.assertEqual(test_errors, [])
            self.assertTrue(res.status.startswith('201 '), res.status)

    def test_PUT_expect_header_nonzero_content_length(self):
        test_errors = []

        def test_connect(ipaddr, port, device, partition, method, path,
                         headers=None, query_string=None):
            if path == '/a/c/o.jpg':
                if 'Expect' not in headers:
                    test_errors.append('Expect was not in headers for '
                                       'non-zero byte PUT!')

        with save_globals():
            controller = \
                proxy_server.ObjectController(self.app, 'a', 'c', 'o.jpg')
            set_http_connect(200, 200, 201, 201, 201,
                             give_connect=test_connect)
            req = Request.blank('/a/c/o.jpg', {})
            req.content_length = 1
            req.body = 'a'
            self.app.update_request(req)
            self.app.memcache.store = {}
            res = controller.PUT(req)
            self.assertTrue(res.status.startswith('201 '))

    def test_expirer_DELETE_on_versioned_object(self):
        test_errors = []

        def test_connect(ipaddr, port, device, partition, method, path,
                         headers=None, query_string=None):
            if method == 'DELETE':
                if 'x-if-delete-at' in headers or 'X-If-Delete-At' in headers:
                    test_errors.append('X-If-Delete-At in headers')

        body = simplejson.dumps(
            [{"name": "001o/1",
              "hash": "x",
              "bytes": 0,
              "content_type": "text/plain",
              "last_modified": "1970-01-01T00:00:01.000000"}])
        body_iter = ('', '', body, '', '', '', '', '', '', '', '', '', '', '')
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
            #                HEAD HEAD GET  GET  HEAD GET  GET  GET  PUT  PUT
            #                PUT  DEL  DEL  DEL
            set_http_connect(200, 200, 200, 200, 200, 200, 200, 200, 201, 201,
                             201, 200, 200, 200,
                             give_connect=test_connect,
                             body_iter=body_iter,
                             headers={'x-versions-location': 'foo'})
            self.app.memcache.store = {}
            req = Request.blank('/a/c/o',
                                headers={'X-If-Delete-At': 1},
                                environ={'REQUEST_METHOD': 'DELETE'})
            self.app.update_request(req)
            res = controller.DELETE(req)
            self.assertEquals(test_errors, [])

    def test_GET_manifest_no_segments(self):
        for hdict in [{"X-Object-Manifest": "segments/seg"},
                      {"X-Static-Large-Object": "True"}]:
            response_bodies = (
                '',                     # HEAD /a
                '',                     # HEAD /a/c
                '',                     # GET manifest
                simplejson.dumps([]))   # GET empty listing

            with save_globals():
                controller = proxy_server.ObjectController(
                    self.app, 'a', 'c', 'manifest')
                set_http_connect(
                    200,    # HEAD /a
                    200,    # HEAD /a/c
                    200,    # GET manifest
                    200,    # GET empty listing
                    headers=hdict,
                    body_iter=response_bodies)

                req = Request.blank('/a/c/manifest')
                resp = controller.GET(req)
                self.assertEqual(resp.status_int, 200)
                self.assertEqual(resp.body, '')

    def test_GET_manifest_limited_listing(self):
        listing1 = [{"hash": "454dfc73af632012ce3e6217dc464241",
                     "last_modified": "2012-11-08T04:05:37.866820",
                     "bytes": 2,
                     "name": "seg01",
                     "content_type": "application/octet-stream"},
                    {"hash": "474bab96c67528d42d5c0c52b35228eb",
                     "last_modified": "2012-11-08T04:05:37.846710",
                     "bytes": 2,
                     "name": "seg02",
                     "content_type": "application/octet-stream"}]

        listing2 = [{"hash": "116baa5508693d1d1ca36abdd9f9478b",
                     "last_modified": "2012-11-08T04:05:37.849510",
                     "bytes": 2,
                     "name": "seg03",
                     "content_type": "application/octet-stream"},
                    {"hash": "7bd6aaa1ef6013353f0420459574ac9d",
                     "last_modified": "2012-11-08T04:05:37.855180",
                     "bytes": 2,
                     "name": "seg04",
                     "content_type": "application/octet-stream"
                     }]

        listing3 = [{"hash": "6605f80e3cefaa24e9823544df4edbd6",
                     "last_modified": "2012-11-08T04:05:37.853710",
                     "bytes": 2,
                     "name": u'\N{SNOWMAN}seg05',
                     "content_type": "application/octet-stream"}]

        response_bodies = (
            '',                           # HEAD /a
            '',                           # HEAD /a/c
            '',                           # GET manifest
            simplejson.dumps(listing1),   # GET listing1
            'Aa',                         # GET seg01
            'Bb',                         # GET seg02
            simplejson.dumps(listing2),   # GET listing2
            'Cc',                         # GET seg03
            'Dd',                         # GET seg04
            simplejson.dumps(listing3),   # GET listing3
            'Ee',                         # GET seg05
            simplejson.dumps([]))         # GET final empty listing
        with save_globals():
            try:
                swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = 2
                controller = proxy_server.ObjectController(
                    self.app, 'a', 'c', 'manifest')

                requested = []

                def capture_requested_paths(ipaddr, port, device, partition,
                                            method, path, headers=None,
                                            query_string=None):
                    qs_dict = dict(urlparse.parse_qsl(query_string or ''))
                    requested.append([method, path, qs_dict])

                set_http_connect(
                    200,    # HEAD /a
                    200,    # HEAD /a/c
                    200,    # GET manifest
                    200,    # GET listing1
                    200,    # GET seg01
                    200,    # GET seg02
                    200,    # GET listing2
                    200,    # GET seg03
                    200,    # GET seg04
                    200,    # GET listing3
                    200,    # GET seg05
                    200,    # GET final empty listing
                    headers={"X-Object-Manifest": "segments/seg"},
                    body_iter=response_bodies,
                    give_connect=capture_requested_paths)

                req = Request.blank('/a/c/manifest')
                resp = controller.GET(req)
                self.assertEqual(resp.status_int, 200)
                self.assertEqual(resp.body, 'AaBbCcDdEe')

                self.assertEqual(
                    requested,
                    [['HEAD', '/a', {}],
                     ['HEAD', '/a/c', {}],
                     ['GET', '/a/c/manifest', {}],
                     ['GET', '/a/segments',
                      {'format': 'json', 'prefix': 'seg'}],
                     ['GET', '/a/segments/seg01', {}],
                     ['GET', '/a/segments/seg02', {}],
                     ['GET', '/a/segments',
                      {'format': 'json', 'prefix': 'seg', 'marker': 'seg02'}],
                     ['GET', '/a/segments/seg03', {}],
                     ['GET', '/a/segments/seg04', {}],
                     ['GET', '/a/segments',
                      {'format': 'json', 'prefix': 'seg', 'marker': 'seg04'}],
                     ['GET', '/a/segments/\xe2\x98\x83seg05', {}],
                     ['GET', '/a/segments',
                      {'format': 'json', 'prefix': 'seg',
                       'marker': '\xe2\x98\x83seg05'}]])

            finally:
                # other tests in this file get very unhappy if this
                # isn't set back, which leads to time-wasting
                # debugging of other tests.
                swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = \
                    _orig_container_listing_limit

    def test_GET_manifest_slo(self):
        listing = [{"hash": "98568d540134639be4655198a36614a4",
                    "last_modified": "2012-11-08T04:05:37.866820",
                    "bytes": 2,
                    "name": "/d1/seg01",
                    "content_type": "application/octet-stream"},
                   {"hash": "d526f1c8ef6c1e4e980e2b8471352d23",
                    "last_modified": "2012-11-08T04:05:37.846710",
                    "bytes": 2,
                    "name": "/d2/seg02",
                    "content_type": "application/octet-stream"}]

        response_bodies = (
            '',                           # HEAD /a
            '',                           # HEAD /a/c
            simplejson.dumps(listing),    # GET manifest
            'Aa',                         # GET seg01
            'Bb')                         # GET seg02
        with save_globals():
            controller = proxy_server.ObjectController(
                self.app, 'a', 'c', 'manifest')

            requested = []

            def capture_requested_paths(ipaddr, port, device, partition,
                                        method, path, headers=None,
                                        query_string=None):
                qs_dict = dict(urlparse.parse_qsl(query_string or ''))
                requested.append([method, path, qs_dict])

            set_http_connect(
                200,    # HEAD /a
                200,    # HEAD /a/c
                200,    # GET listing1
                200,    # GET seg01
                200,    # GET seg02
                headers=[{}, {}, {"X-Static-Large-Object": "True",
                         'content-type': 'text/html; swift_bytes=4'}, {}, {}],
                body_iter=response_bodies,
                give_connect=capture_requested_paths)

            req = Request.blank('/a/c/manifest')
            resp = controller.GET(req)
            self.assertEqual(resp.status_int, 200)
            self.assertEqual(resp.body, 'AaBb')
            self.assertEqual(resp.content_length, 4)
            self.assertEqual(resp.content_type, 'text/html')

            self.assertEqual(
                requested,
                [['HEAD', '/a', {}],
                 ['HEAD', '/a/c', {}],
                 ['GET', '/a/c/manifest', {}],
                 ['GET', '/a/d1/seg01', {}],
                 ['GET', '/a/d2/seg02', {}]])

    def test_GET_slo_multipart_manifest(self):
        listing = [{"hash": "98568d540134639be4655198a36614a4",
                    "last_modified": "2012-11-08T04:05:37.866820",
                    "bytes": 2,
                    "name": "/d1/seg01",
                    "content_type": "application/octet-stream"},
                   {"hash": "d526f1c8ef6c1e4e980e2b8471352d23",
                    "last_modified": "2012-11-08T04:05:37.846710",
                    "bytes": 2,
                    "name": "/d2/seg02",
                    "content_type": "application/octet-stream"}]
        json_listing = simplejson.dumps(listing)
        response_bodies = (
            '',                           # HEAD /a
            '',                           # HEAD /a/c
            json_listing)    # GET manifest
        with save_globals():
            controller = proxy_server.ObjectController(
                self.app, 'a', 'c', 'manifest')

            requested = []

            def capture_requested_paths(ipaddr, port, device, partition,
                                        method, path, headers=None,
                                        query_string=None):
                qs_dict = dict(urlparse.parse_qsl(query_string or ''))
                requested.append([method, path, qs_dict])

            set_http_connect(
                200,    # HEAD /a
                200,    # HEAD /a/c
                200,    # GET listing1
                headers={"X-Static-Large-Object": "True",
                         'content-type': 'text/html; swift_bytes=4'},
                body_iter=response_bodies,
                give_connect=capture_requested_paths)

            req = Request.blank('/a/c/manifest?multipart-manifest=get')
            resp = controller.GET(req)
            self.assertEqual(resp.status_int, 200)
            self.assertEqual(resp.body, json_listing)
            self.assertEqual(resp.content_type, 'application/json')

            self.assertEqual(
                requested,
                [['HEAD', '/a', {}],
                 ['HEAD', '/a/c', {}],
                 ['GET', '/a/c/manifest', {'multipart-manifest': 'get'}]])

    def test_GET_slo_multipart_manifest_from_copy(self):
        listing = [{"hash": "98568d540134639be4655198a36614a4",
                    "last_modified": "2012-11-08T04:05:37.866820",
                    "bytes": 2,
                    "name": "/d1/seg01",
                    "content_type": "application/octet-stream"},
                   {"hash": "d526f1c8ef6c1e4e980e2b8471352d23",
                    "last_modified": "2012-11-08T04:05:37.846710",
                    "bytes": 2,
                    "name": "/d2/seg02",
                    "content_type": "application/octet-stream"}]
        json_listing = simplejson.dumps(listing)
        response_bodies = (
            '',                           # HEAD /a
            '',                           # HEAD /a/c
            json_listing)    # GET manifest
        with save_globals():
            controller = proxy_server.ObjectController(
                self.app, 'a', 'c', 'manifest')

            requested = []

            def capture_requested_paths(ipaddr, port, device, partition,
                                        method, path, headers=None,
                                        query_string=None):
                qs_dict = dict(urlparse.parse_qsl(query_string or ''))
                requested.append([method, path, qs_dict])

            set_http_connect(
                200,    # HEAD /a
                200,    # HEAD /a/c
                200,    # GET listing1
                headers={"X-Static-Large-Object": "True",
                         'content-type': 'text/html; swift_bytes=4'},
                body_iter=response_bodies,
                give_connect=capture_requested_paths)

            req = Request.blank('/a/c/manifest?multipart-manifest=get',
                                headers={'x-copy-from': '/a/c/manifest'})
            resp = controller.GET(req)
            self.assertEqual(resp.status_int, 200)
            self.assertEqual(resp.body, json_listing)
            self.assertEqual(resp.content_type, 'text/html')

            self.assertEqual(
                requested,
                [['HEAD', '/a', {}],
                 ['HEAD', '/a/c', {}],
                 ['GET', '/a/c/manifest', {'multipart-manifest': 'get'}]])

    def test_GET_bad_etag_manifest_slo(self):
        listing = [{"hash": "98568d540134639be4655198a36614a4",
                    "last_modified": "2012-11-08T04:05:37.866820",
                    "bytes": 2,
                    "name": "/d1/seg01",
                    "content_type": "application/octet-stream"},
                   {"hash": "invalidhash",
                    "last_modified": "2012-11-08T04:05:37.846710",
                    "bytes": 2,
                    "name": "/d2/seg02",
                    "content_type": "application/octet-stream"}]

        response_bodies = (
            '',                           # HEAD /a
            '',                           # HEAD /a/c
            simplejson.dumps(listing),    # GET manifest
            'Aa',                         # GET seg01
            'Bb')                         # GET seg02
        with save_globals():
            controller = proxy_server.ObjectController(
                self.app, 'a', 'c', 'manifest')

            requested = []

            def capture_requested_paths(ipaddr, port, device, partition,
                                        method, path, headers=None,
                                        query_string=None):
                qs_dict = dict(urlparse.parse_qsl(query_string or ''))
                requested.append([method, path, qs_dict])

            set_http_connect(
                200,    # HEAD /a
                200,    # HEAD /a/c
                200,    # GET listing1
                200,    # GET seg01
                200,    # GET seg02
                headers=[{}, {}, {"X-Static-Large-Object": "True",
                         'content-type': 'text/html; swift_bytes=4'}, {}, {}],
                body_iter=response_bodies,
                give_connect=capture_requested_paths)
            req = Request.blank('/a/c/manifest')
            resp = controller.GET(req)
            self.assertEqual(resp.status_int, 200)
            self.assertEqual(resp.body, 'Aa')  # dropped connection
            self.assertEqual(resp.content_length, 4)  # content incomplete
            self.assertEqual(resp.content_type, 'text/html')

            self.assertEqual(
                requested,
                [['HEAD', '/a', {}],
                 ['HEAD', '/a/c', {}],
                 ['GET', '/a/c/manifest', {}],
                 ['GET', '/a/d1/seg01', {}],
                 ['GET', '/a/d2/seg02', {}]])

    def test_GET_nested_slo(self):
        listing = [{"hash": "98568d540134639be4655198a36614a4",
                    "last_modified": "2012-11-08T04:05:37.866820",
                    "bytes": 2,
                    "name": "/d1/seg01",
                    "content_type": "application/octet-stream"},
                   {"hash": "d526f1c8ef6c1e4e980e2b8471352d23",
                    "last_modified": "2012-11-08T04:05:37.846710",
                    "bytes": 2,
                    "name": "/d2/seg02",
                    "content_type": "application/octet-stream"}]

        response_bodies = (
            '',                           # HEAD /a
            '',                           # HEAD /a/c
            simplejson.dumps(listing),    # GET manifest
            'Aa',                         # GET seg01
            'Bb')                         # GET seg02
        with save_globals():
            controller = proxy_server.ObjectController(
                self.app, 'a', 'c', 'manifest')

            requested = []

            def capture_requested_paths(ipaddr, port, device, partition,
                                        method, path, headers=None,
                                        query_string=None):
                qs_dict = dict(urlparse.parse_qsl(query_string or ''))
                requested.append([method, path, qs_dict])

            slob_headers = {"X-Static-Large-Object": "True",
                            'content-type': 'text/html; swift_bytes=4'}
            set_http_connect(
                200,    # HEAD /a
                200,    # HEAD /a/c
                200,    # GET listing1
                200,    # GET seg01
                200,    # GET seg02
                headers=[{}, {}, slob_headers, {}, slob_headers],
                body_iter=response_bodies,
                give_connect=capture_requested_paths)
            req = Request.blank('/a/c/manifest')
            resp = controller.GET(req)
            self.assertEqual(resp.status_int, 200)
            self.assertEqual(resp.body, 'Aa')  # dropped connection
            self.assertEqual(resp.content_length, 4)  # content incomplete
            self.assertEqual(resp.content_type, 'text/html')

            self.assertEqual(
                requested,
                [['HEAD', '/a', {}],
                 ['HEAD', '/a/c', {}],
                 ['GET', '/a/c/manifest', {}],
                 ['GET', '/a/d1/seg01', {}],
                 ['GET', '/a/d2/seg02', {}]])

    def test_GET_bad_404_manifest_slo(self):
        listing = [{"hash": "98568d540134639be4655198a36614a4",
                    "last_modified": "2012-11-08T04:05:37.866820",
                    "bytes": 2,
                    "name": "/d1/seg01",
                    "content_type": "application/octet-stream"},
                   {"hash": "d526f1c8ef6c1e4e980e2b8471352d23",
                    "last_modified": "2012-11-08T04:05:37.846710",
                    "bytes": 2,
                    "name": "/d2/seg02",
                    "content_type": "application/octet-stream"},
                   {"hash": "invalidhash",
                    "last_modified": "2012-11-08T04:05:37.846710",
                    "bytes": 2,
                    "name": "/d2/seg03",
                    "content_type": "application/octet-stream"}]

        response_bodies = (
            '',                           # HEAD /a
            '',                           # HEAD /a/c
            simplejson.dumps(listing),    # GET manifest
            'Aa',                         # GET seg01
            '')                           # GET seg02
        with save_globals():
            controller = proxy_server.ObjectController(
                self.app, 'a', 'c', 'manifest')

            requested = []

            def capture_requested_paths(ipaddr, port, device, partition,
                                        method, path, headers=None,
                                        query_string=None):
                qs_dict = dict(urlparse.parse_qsl(query_string or ''))
                requested.append([method, path, qs_dict])

            set_http_connect(
                200,    # HEAD /a
                200,    # HEAD /a/c
                200,    # GET listing1
                200,    # GET seg01
                404,    # GET seg02
                headers=[{}, {}, {"X-Static-Large-Object": "True",
                         'content-type': 'text/html; swift_bytes=4'}, {}, {}],
                body_iter=response_bodies,
                give_connect=capture_requested_paths)
            req = Request.blank('/a/c/manifest')
            resp = controller.GET(req)
            self.assertEqual(resp.status_int, 200)
            self.assertEqual(resp.body, 'Aa')  # dropped connection
            self.assertEqual(resp.content_length, 6)  # content incomplete
            self.assertEqual(resp.content_type, 'text/html')

            self.assertEqual(
                requested,
                [['HEAD', '/a', {}],
                 ['HEAD', '/a/c', {}],
                 ['GET', '/a/c/manifest', {}],
                 ['GET', '/a/d1/seg01', {}],
                 ['GET', '/a/d2/seg02', {}],
                 ['GET', '/a/d2/seg02', {}],
                 ['GET', '/a/d2/seg02', {}]])  # 2nd segment not found

    def test_HEAD_manifest_slo(self):
        listing = [{"hash": "454dfc73af632012ce3e6217dc464241",
                    "last_modified": "2012-11-08T04:05:37.866820",
                    "bytes": 2,
                    "name": "/d1/seg01",
                    "content_type": "application/octet-stream"},
                   {"hash": "474bab96c67528d42d5c0c52b35228eb",
                    "last_modified": "2012-11-08T04:05:37.846710",
                    "bytes": 2,
                    "name": "/d2/seg02",
                    "content_type": "application/octet-stream"}]

        response_bodies = (
            '',                              # HEAD /a
            '',                              # HEAD /a/c
            '',                              # HEAD manifest
            simplejson.dumps(listing))       # GET manifest
        with save_globals():
            controller = proxy_server.ObjectController(
                self.app, 'a', 'c', 'manifest')

            requested = []

            def capture_requested_paths(ipaddr, port, device, partition,
                                        method, path, headers=None,
                                        query_string=None):
                qs_dict = dict(urlparse.parse_qsl(query_string or ''))
                requested.append([method, path, qs_dict])

            set_http_connect(
                200,    # HEAD /a
                200,    # HEAD /a/c
                200,    # HEAD listing1
                200,    # GET listing1
                headers={"X-Static-Large-Object": "True"},
                body_iter=response_bodies,
                give_connect=capture_requested_paths)

            req = Request.blank('/a/c/manifest',
                                environ={'REQUEST_METHOD': 'HEAD'})
            resp = controller.HEAD(req)
            self.assertEqual(resp.status_int, 200)

            self.assertEqual(
                requested,
                [['HEAD', '/a', {}],
                 ['HEAD', '/a/c', {}],
                 ['HEAD', '/a/c/manifest', {}],
                 ['GET', '/a/c/manifest', {}]])

    def test_PUT_auto_content_type(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')

            def test_content_type(filename, expected):
                # The three responses here are for account_info() (HEAD to
                # account server), container_info() (HEAD to container server)
                # and three calls to _connect_put_node() (PUT to three object
                # servers)
                set_http_connect(201, 201, 201, 201, 201,
                                 give_content_type=lambda content_type:
                                 self.assertEquals(content_type,
                                                   expected.next()))
                # We need into include a transfer-encoding to get past
                # constraints.check_object_creation()
                req = Request.blank('/a/c/%s' % filename, {},
                                    headers={'transfer-encoding': 'chunked'})
                self.app.update_request(req)
                self.app.memcache.store = {}
                res = controller.PUT(req)
                # If we don't check the response here we could miss problems
                # in PUT()
                self.assertEquals(res.status_int, 201)

            test_content_type('test.jpg', iter(['', '', 'image/jpeg',
                                                'image/jpeg', 'image/jpeg']))
            test_content_type('test.html', iter(['', '', 'text/html',
                                                 'text/html', 'text/html']))
            test_content_type('test.css', iter(['', '', 'text/css',
                                                'text/css', 'text/css']))

    def test_custom_mime_types_files(self):
        swift_dir = mkdtemp()
        try:
            with open(os.path.join(swift_dir, 'mime.types'), 'w') as fp:
                fp.write('foo/bar foo\n')
            ba = proxy_server.Application({'swift_dir': swift_dir},
                                          FakeMemcache(), FakeLogger(),
                                          FakeRing(), FakeRing(),
                                          FakeRing())
            self.assertEquals(proxy_server.mimetypes.guess_type('blah.foo')[0],
                              'foo/bar')
            self.assertEquals(proxy_server.mimetypes.guess_type('blah.jpg')[0],
                              'image/jpeg')
        finally:
            rmtree(swift_dir, ignore_errors=True)

    def test_PUT(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')

            def test_status_map(statuses, expected):
                set_http_connect(*statuses)
                req = Request.blank('/a/c/o.jpg', {})
                req.content_length = 0
                self.app.update_request(req)
                self.app.memcache.store = {}
                res = controller.PUT(req)
                expected = str(expected)
                self.assertEquals(res.status[:len(expected)], expected)
            test_status_map((200, 200, 201, 201, 201), 201)
            test_status_map((200, 200, 201, 201, 500), 201)
            test_status_map((200, 200, 204, 404, 404), 404)
            test_status_map((200, 200, 204, 500, 404), 503)

    def test_PUT_connect_exceptions(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')

            def test_status_map(statuses, expected):
                set_http_connect(*statuses)
                self.app.memcache.store = {}
                req = Request.blank('/a/c/o.jpg', {})
                req.content_length = 0
                self.app.update_request(req)
                res = controller.PUT(req)
                expected = str(expected)
                self.assertEquals(res.status[:len(expected)], expected)
            test_status_map((200, 200, 201, 201, -1), 201)
            test_status_map((200, 200, 201, 201, -2), 201)  # expect timeout
            test_status_map((200, 200, 201, 201, -3), 201)  # error limited
            test_status_map((200, 200, 201, -1, -1), 503)
            test_status_map((200, 200, 503, 503, -1), 503)

    def test_PUT_send_exceptions(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')

            def test_status_map(statuses, expected):
                self.app.memcache.store = {}
                set_http_connect(*statuses)
                req = Request.blank('/a/c/o.jpg',
                                    environ={'REQUEST_METHOD': 'PUT'},
                                    body='some data')
                self.app.update_request(req)
                res = controller.PUT(req)
                expected = str(expected)
                self.assertEquals(res.status[:len(expected)], expected)
            test_status_map((200, 200, 201, -1, 201), 201)
            test_status_map((200, 200, 201, -1, -1), 503)
            test_status_map((200, 200, 503, 503, -1), 503)

    def test_PUT_max_size(self):
        with save_globals():
            set_http_connect(201, 201, 201)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            req = Request.blank('/a/c/o', {}, headers={
                'Content-Length': str(MAX_FILE_SIZE + 1),
                'Content-Type': 'foo/bar'})
            self.app.update_request(req)
            res = controller.PUT(req)
            self.assertEquals(res.status_int, 413)

    def test_PUT_bad_content_type(self):
        with save_globals():
            set_http_connect(201, 201, 201)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            req = Request.blank('/a/c/o', {}, headers={
                'Content-Length': 0, 'Content-Type': 'foo/bar;swift_hey=45'})
            self.app.update_request(req)
            res = controller.PUT(req)
            self.assertEquals(res.status_int, 400)

    def test_PUT_getresponse_exceptions(self):

        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')

            def test_status_map(statuses, expected):
                self.app.memcache.store = {}
                set_http_connect(*statuses)
                req = Request.blank('/a/c/o.jpg', {})
                req.content_length = 0
                self.app.update_request(req)
                res = controller.PUT(req)
                expected = str(expected)
                self.assertEquals(res.status[:len(str(expected))],
                                  str(expected))
            test_status_map((200, 200, 201, 201, -1), 201)
            test_status_map((200, 200, 201, -1, -1), 503)
            test_status_map((200, 200, 503, 503, -1), 503)

    def test_POST(self):
        with save_globals():
            self.app.object_post_as_copy = False
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')

            def test_status_map(statuses, expected):
                set_http_connect(*statuses)
                self.app.memcache.store = {}
                req = Request.blank('/a/c/o', {},
                                    headers={'Content-Type': 'foo/bar'})
                self.app.update_request(req)
                res = controller.POST(req)
                expected = str(expected)
                self.assertEquals(res.status[:len(expected)], expected)
            test_status_map((200, 200, 202, 202, 202), 202)
            test_status_map((200, 200, 202, 202, 500), 202)
            test_status_map((200, 200, 202, 500, 500), 503)
            test_status_map((200, 200, 202, 404, 500), 503)
            test_status_map((200, 200, 202, 404, 404), 404)
            test_status_map((200, 200, 404, 500, 500), 503)
            test_status_map((200, 200, 404, 404, 404), 404)

    def test_POST_as_copy(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')

            def test_status_map(statuses, expected):
                set_http_connect(*statuses)
                self.app.memcache.store = {}
                req = Request.blank('/a/c/o', {},
                                    headers={'Content-Type': 'foo/bar'})
                self.app.update_request(req)
                res = controller.POST(req)
                expected = str(expected)
                self.assertEquals(res.status[:len(expected)], expected)
            test_status_map((200, 200, 200, 200, 200, 202, 202, 202), 202)
            test_status_map((200, 200, 200, 200, 200, 202, 202, 500), 202)
            test_status_map((200, 200, 200, 200, 200, 202, 500, 500), 503)
            test_status_map((200, 200, 200, 200, 200, 202, 404, 500), 503)
            test_status_map((200, 200, 200, 200, 200, 202, 404, 404), 404)
            test_status_map((200, 200, 200, 200, 200, 404, 500, 500), 503)
            test_status_map((200, 200, 200, 200, 200, 404, 404, 404), 404)

    def test_DELETE(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')

            def test_status_map(statuses, expected):
                set_http_connect(*statuses)
                self.app.memcache.store = {}
                req = Request.blank('/a/c/o', {})
                self.app.update_request(req)
                res = controller.DELETE(req)
                self.assertEquals(res.status[:len(str(expected))],
                                  str(expected))
            test_status_map((200, 200, 204, 204, 204), 204)
            test_status_map((200, 200, 204, 204, 500), 204)
            test_status_map((200, 200, 204, 404, 404), 404)
            test_status_map((200, 200, 204, 500, 404), 503)
            test_status_map((200, 200, 404, 404, 404), 404)
            test_status_map((200, 200, 404, 404, 500), 404)

    def test_HEAD(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')

            def test_status_map(statuses, expected):
                set_http_connect(*statuses)
                self.app.memcache.store = {}
                req = Request.blank('/a/c/o', {})
                self.app.update_request(req)
                res = controller.HEAD(req)
                self.assertEquals(res.status[:len(str(expected))],
                                  str(expected))
                if expected < 400:
                    self.assert_('x-works' in res.headers)
                    self.assertEquals(res.headers['x-works'], 'yes')
                    self.assert_('accept-ranges' in res.headers)
                    self.assertEquals(res.headers['accept-ranges'], 'bytes')

            test_status_map((200, 200, 200, 404, 404), 200)
            test_status_map((200, 200, 200, 500, 404), 200)
            test_status_map((200, 200, 304, 500, 404), 304)
            test_status_map((200, 200, 404, 404, 404), 404)
            test_status_map((200, 200, 404, 404, 500), 404)
            test_status_map((200, 200, 500, 500, 500), 503)

    def test_HEAD_newest(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')

            def test_status_map(statuses, expected, timestamps,
                                expected_timestamp):
                set_http_connect(*statuses, timestamps=timestamps)
                self.app.memcache.store = {}
                req = Request.blank('/a/c/o', {}, headers={'x-newest': 'true'})
                self.app.update_request(req)
                res = controller.HEAD(req)
                self.assertEquals(res.status[:len(str(expected))],
                                  str(expected))
                self.assertEquals(res.headers.get('last-modified'),
                                  expected_timestamp)

            #                acct cont obj  obj  obj
            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
                                                             '2', '3'), '3')
            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
                                                             '3', '2'), '3')
            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
                                                             '3', '1'), '3')
            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3',
                                                             '3', '1'), '3')
            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None,
                                                             None, None), None)
            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None,
                                                             None, '1'), '1')

    def test_GET_newest(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')

            def test_status_map(statuses, expected, timestamps,
                                expected_timestamp):
                set_http_connect(*statuses, timestamps=timestamps)
                self.app.memcache.store = {}
                req = Request.blank('/a/c/o', {}, headers={'x-newest': 'true'})
                self.app.update_request(req)
                res = controller.GET(req)
                self.assertEquals(res.status[:len(str(expected))],
                                  str(expected))
                self.assertEquals(res.headers.get('last-modified'),
                                  expected_timestamp)

            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
                                                             '2', '3'), '3')
            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
                                                             '3', '2'), '3')
            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
                                                             '3', '1'), '3')
            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3',
                                                             '3', '1'), '3')
            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None,
                                                             None, None), None)
            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None,
                                                             None, '1'), '1')

        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')

            def test_status_map(statuses, expected, timestamps,
                                expected_timestamp):
                set_http_connect(*statuses, timestamps=timestamps)
                self.app.memcache.store = {}
                req = Request.blank('/a/c/o', {})
                self.app.update_request(req)
                res = controller.HEAD(req)
                self.assertEquals(res.status[:len(str(expected))],
                                  str(expected))
                self.assertEquals(res.headers.get('last-modified'),
                                  expected_timestamp)

            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
                                                             '2', '3'), '1')
            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
                                                             '3', '2'), '1')
            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
                                                             '3', '1'), '1')
            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3',
                                                             '3', '1'), '3')
            test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None,
                                                             '1', '2'), None)

    def test_POST_meta_val_len(self):
        with save_globals():
            limit = MAX_META_VALUE_LENGTH
            self.app.object_post_as_copy = False
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 202, 202, 202)
            #                acct cont obj  obj  obj
            req = Request.blank('/a/c/o', {}, headers={
                                'Content-Type': 'foo/bar',
                                'X-Object-Meta-Foo': 'x' * limit})
            self.app.update_request(req)
            res = controller.POST(req)
            self.assertEquals(res.status_int, 202)
            set_http_connect(202, 202, 202)
            req = Request.blank('/a/c/o', {}, headers={
                                'Content-Type': 'foo/bar',
                                'X-Object-Meta-Foo': 'x' * (limit + 1)})
            self.app.update_request(req)
            res = controller.POST(req)
            self.assertEquals(res.status_int, 400)

    def test_POST_as_copy_meta_val_len(self):
        with save_globals():
            limit = MAX_META_VALUE_LENGTH
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 200, 200, 200, 202, 202, 202)
            #                acct cont objc objc objc obj  obj  obj
            req = Request.blank('/a/c/o', {}, headers={
                                'Content-Type': 'foo/bar',
                                'X-Object-Meta-Foo': 'x' * limit})
            self.app.update_request(req)
            res = controller.POST(req)
            self.assertEquals(res.status_int, 202)
            set_http_connect(202, 202, 202)
            req = Request.blank('/a/c/o', {}, headers={
                                'Content-Type': 'foo/bar',
                                'X-Object-Meta-Foo': 'x' * (limit + 1)})
            self.app.update_request(req)
            res = controller.POST(req)
            self.assertEquals(res.status_int, 400)

    def test_POST_meta_key_len(self):
        with save_globals():
            limit = MAX_META_NAME_LENGTH
            self.app.object_post_as_copy = False
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 202, 202, 202)
            #                acct cont obj  obj  obj
            req = Request.blank('/a/c/o', {}, headers={
                'Content-Type': 'foo/bar',
                ('X-Object-Meta-' + 'x' * limit): 'x'})
            self.app.update_request(req)
            res = controller.POST(req)
            self.assertEquals(res.status_int, 202)
            set_http_connect(202, 202, 202)
            req = Request.blank('/a/c/o', {}, headers={
                'Content-Type': 'foo/bar',
                ('X-Object-Meta-' + 'x' * (limit + 1)): 'x'})
            self.app.update_request(req)
            res = controller.POST(req)
            self.assertEquals(res.status_int, 400)

    def test_POST_as_copy_meta_key_len(self):
        with save_globals():
            limit = MAX_META_NAME_LENGTH
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 200, 200, 200, 202, 202, 202)
            #                acct cont objc objc objc obj  obj  obj
            req = Request.blank('/a/c/o', {}, headers={
                'Content-Type': 'foo/bar',
                ('X-Object-Meta-' + 'x' * limit): 'x'})
            self.app.update_request(req)
            res = controller.POST(req)
            self.assertEquals(res.status_int, 202)
            set_http_connect(202, 202, 202)
            req = Request.blank('/a/c/o', {}, headers={
                'Content-Type': 'foo/bar',
                ('X-Object-Meta-' + 'x' * (limit + 1)): 'x'})
            self.app.update_request(req)
            res = controller.POST(req)
            self.assertEquals(res.status_int, 400)

    def test_POST_meta_count(self):
        with save_globals():
            limit = MAX_META_COUNT
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            headers = dict(
                (('X-Object-Meta-' + str(i), 'a') for i in xrange(limit + 1)))
            headers.update({'Content-Type': 'foo/bar'})
            set_http_connect(202, 202, 202)
            req = Request.blank('/a/c/o', {}, headers=headers)
            self.app.update_request(req)
            res = controller.POST(req)
            self.assertEquals(res.status_int, 400)

    def test_POST_meta_size(self):
        with save_globals():
            limit = MAX_META_OVERALL_SIZE
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            count = limit / 256  # enough to cause the limit to be reached
            headers = dict(
                (('X-Object-Meta-' + str(i), 'a' * 256)
                    for i in xrange(count + 1)))
            headers.update({'Content-Type': 'foo/bar'})
            set_http_connect(202, 202, 202)
            req = Request.blank('/a/c/o', {}, headers=headers)
            self.app.update_request(req)
            res = controller.POST(req)
            self.assertEquals(res.status_int, 400)

    def test_client_timeout(self):
        with save_globals():
            self.app.account_ring.get_nodes('account')
            for dev in self.app.account_ring.devs.values():
                dev['ip'] = '127.0.0.1'
                dev['port'] = 1
            self.app.container_ring.get_nodes('account')
            for dev in self.app.container_ring.devs.values():
                dev['ip'] = '127.0.0.1'
                dev['port'] = 1
            self.app.object_ring.get_nodes('account')
            for dev in self.app.object_ring.devs.values():
                dev['ip'] = '127.0.0.1'
                dev['port'] = 1

            class SlowBody():

                def __init__(self):
                    self.sent = 0

                def read(self, size=-1):
                    if self.sent < 4:
                        sleep(0.1)
                        self.sent += 1
                        return ' '
                    return ''

            req = Request.blank('/a/c/o',
                                environ={'REQUEST_METHOD': 'PUT',
                                         'wsgi.input': SlowBody()},
                                headers={'Content-Length': '4',
                                         'Content-Type': 'text/plain'})
            self.app.update_request(req)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 201, 201, 201)
            #                acct cont obj  obj  obj
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)
            self.app.client_timeout = 0.1
            req = Request.blank('/a/c/o',
                                environ={'REQUEST_METHOD': 'PUT',
                                         'wsgi.input': SlowBody()},
                                headers={'Content-Length': '4',
                                         'Content-Type': 'text/plain'})
            self.app.update_request(req)
            set_http_connect(201, 201, 201)
            #                obj  obj  obj
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 408)

    def test_client_disconnect(self):
        with save_globals():
            self.app.account_ring.get_nodes('account')
            for dev in self.app.account_ring.devs.values():
                dev['ip'] = '127.0.0.1'
                dev['port'] = 1
            self.app.container_ring.get_nodes('account')
            for dev in self.app.container_ring.devs.values():
                dev['ip'] = '127.0.0.1'
                dev['port'] = 1
            self.app.object_ring.get_nodes('account')
            for dev in self.app.object_ring.devs.values():
                dev['ip'] = '127.0.0.1'
                dev['port'] = 1

            class SlowBody():

                def __init__(self):
                    self.sent = 0

                def read(self, size=-1):
                    raise Exception('Disconnected')

            req = Request.blank('/a/c/o',
                                environ={'REQUEST_METHOD': 'PUT',
                                         'wsgi.input': SlowBody()},
                                headers={'Content-Length': '4',
                                         'Content-Type': 'text/plain'})
            self.app.update_request(req)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 201, 201, 201)
            #                acct cont obj  obj  obj
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 499)

    def test_node_read_timeout(self):
        with save_globals():
            self.app.account_ring.get_nodes('account')
            for dev in self.app.account_ring.devs.values():
                dev['ip'] = '127.0.0.1'
                dev['port'] = 1
            self.app.container_ring.get_nodes('account')
            for dev in self.app.container_ring.devs.values():
                dev['ip'] = '127.0.0.1'
                dev['port'] = 1
            self.app.object_ring.get_nodes('account')
            for dev in self.app.object_ring.devs.values():
                dev['ip'] = '127.0.0.1'
                dev['port'] = 1
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'})
            self.app.update_request(req)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 200, slow=True)
            req.sent_size = 0
            resp = controller.GET(req)
            got_exc = False
            try:
                resp.body
            except ChunkReadTimeout:
                got_exc = True
            self.assert_(not got_exc)
            self.app.node_timeout = 0.1
            set_http_connect(200, 200, 200, slow=True)
            resp = controller.GET(req)
            got_exc = False
            try:
                resp.body
            except ChunkReadTimeout:
                got_exc = True
            self.assert_(got_exc)

    def test_node_write_timeout(self):
        with save_globals():
            self.app.account_ring.get_nodes('account')
            for dev in self.app.account_ring.devs.values():
                dev['ip'] = '127.0.0.1'
                dev['port'] = 1
            self.app.container_ring.get_nodes('account')
            for dev in self.app.container_ring.devs.values():
                dev['ip'] = '127.0.0.1'
                dev['port'] = 1
            self.app.object_ring.get_nodes('account')
            for dev in self.app.object_ring.devs.values():
                dev['ip'] = '127.0.0.1'
                dev['port'] = 1
            req = Request.blank('/a/c/o',
                                environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '4',
                                         'Content-Type': 'text/plain'},
                                body='    ')
            self.app.update_request(req)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 201, 201, 201, slow=True)
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)
            self.app.node_timeout = 0.1
            set_http_connect(201, 201, 201, slow=True)
            req = Request.blank('/a/c/o',
                                environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '4',
                                         'Content-Type': 'text/plain'},
                                body='    ')
            self.app.update_request(req)
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 503)

    def test_iter_nodes(self):
        with save_globals():
            try:
                self.app.object_ring.max_more_nodes = 2
                controller = proxy_server.ObjectController(self.app, 'account',
                                                           'container',
                                                           'object')
                partition, nodes = self.app.object_ring.get_nodes('account',
                                                                  'container',
                                                                  'object')
                collected_nodes = []
                for node in controller.iter_nodes(self.app.object_ring,
                                                  partition):
                    collected_nodes.append(node)
                self.assertEquals(len(collected_nodes), 5)

                self.app.object_ring.max_more_nodes = 20
                self.app.request_node_count = lambda r: 20
                controller = proxy_server.ObjectController(self.app, 'account',
                                                           'container',
                                                           'object')
                partition, nodes = self.app.object_ring.get_nodes('account',
                                                                  'container',
                                                                  'object')
                collected_nodes = []
                for node in controller.iter_nodes(self.app.object_ring,
                                                  partition):
                    collected_nodes.append(node)
                self.assertEquals(len(collected_nodes), 9)

                self.app.log_handoffs = True
                self.app.logger = FakeLogger()
                self.app.object_ring.max_more_nodes = 2
                controller = proxy_server.ObjectController(self.app, 'account',
                                                           'container',
                                                           'object')
                partition, nodes = self.app.object_ring.get_nodes('account',
                                                                  'container',
                                                                  'object')
                collected_nodes = []
                for node in controller.iter_nodes(self.app.object_ring,
                                                  partition):
                    collected_nodes.append(node)
                self.assertEquals(len(collected_nodes), 5)
                self.assertEquals(
                    self.app.logger.log_dict['warning'],
                    [(('Handoff requested (1)',), {}),
                     (('Handoff requested (2)',), {})])

                self.app.log_handoffs = False
                self.app.logger = FakeLogger()
                self.app.object_ring.max_more_nodes = 2
                controller = proxy_server.ObjectController(self.app, 'account',
                                                           'container',
                                                           'object')
                partition, nodes = self.app.object_ring.get_nodes('account',
                                                                  'container',
                                                                  'object')
                collected_nodes = []
                for node in controller.iter_nodes(self.app.object_ring,
                                                  partition):
                    collected_nodes.append(node)
                self.assertEquals(len(collected_nodes), 5)
                self.assertEquals(self.app.logger.log_dict['warning'], [])
            finally:
                self.app.object_ring.max_more_nodes = 0

    def test_iter_nodes_calls_sort_nodes(self):
        with mock.patch.object(self.app, 'sort_nodes') as sort_nodes:
            controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
            for node in controller.iter_nodes(self.app.object_ring, 0):
                pass
            sort_nodes.assert_called_once_with(
                self.app.object_ring.get_part_nodes(0))

    def test_iter_nodes_skips_error_limited(self):
        with mock.patch.object(self.app, 'sort_nodes', lambda n: n):
            controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
            first_nodes = list(controller.iter_nodes(self.app.object_ring, 0))
            second_nodes = list(controller.iter_nodes(self.app.object_ring, 0))
            self.assertTrue(first_nodes[0] in second_nodes)

            controller.error_limit(first_nodes[0], 'test')
            second_nodes = list(controller.iter_nodes(self.app.object_ring, 0))
            self.assertTrue(first_nodes[0] not in second_nodes)

    def test_iter_nodes_gives_extra_if_error_limited_inline(self):
        with nested(
                mock.patch.object(self.app, 'sort_nodes', lambda n: n),
                mock.patch.object(self.app, 'request_node_count',
                                  lambda r: 6),
                mock.patch.object(self.app.object_ring, 'max_more_nodes', 99)):
            controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
            first_nodes = list(controller.iter_nodes(self.app.object_ring, 0))
            second_nodes = []
            for node in controller.iter_nodes(self.app.object_ring, 0):
                if not second_nodes:
                    controller.error_limit(node, 'test')
                second_nodes.append(node)
            self.assertEquals(len(first_nodes), 6)
            self.assertEquals(len(second_nodes), 7)

    def test_best_response_sets_etag(self):
        controller = proxy_server.ObjectController(self.app, 'account',
                                                   'container', 'object')
        req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'})
        resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3,
                                        'Object')
        self.assertEquals(resp.etag, None)
        resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3,
                                        'Object',
                                        etag='68b329da9893e34099c7d8ad5cb9c940'
                                        )
        self.assertEquals(resp.etag, '68b329da9893e34099c7d8ad5cb9c940')

    def test_proxy_passes_content_type(self):
        with save_globals():
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'})
            self.app.update_request(req)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 200)
            resp = controller.GET(req)
            self.assertEquals(resp.status_int, 200)
            self.assertEquals(resp.content_type, 'x-application/test')
            set_http_connect(200, 200, 200)
            resp = controller.GET(req)
            self.assertEquals(resp.status_int, 200)
            self.assertEquals(resp.content_length, 0)
            set_http_connect(200, 200, 200, slow=True)
            resp = controller.GET(req)
            self.assertEquals(resp.status_int, 200)
            self.assertEquals(resp.content_length, 4)

    def test_proxy_passes_content_length_on_head(self):
        with save_globals():
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'HEAD'})
            self.app.update_request(req)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 200)
            resp = controller.HEAD(req)
            self.assertEquals(resp.status_int, 200)
            self.assertEquals(resp.content_length, 0)
            set_http_connect(200, 200, 200, slow=True)
            resp = controller.HEAD(req)
            self.assertEquals(resp.status_int, 200)
            self.assertEquals(resp.content_length, 4)

    def test_error_limiting(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            controller.app.sort_nodes = lambda l: l
            self.assert_status_map(controller.HEAD, (200, 200, 503, 200, 200),
                                   200)
            self.assertEquals(controller.app.object_ring.devs[0]['errors'], 2)
            self.assert_('last_error' in controller.app.object_ring.devs[0])
            for _junk in xrange(self.app.error_suppression_limit):
                self.assert_status_map(controller.HEAD, (200, 200, 503, 503,
                                                         503), 503)
            self.assertEquals(controller.app.object_ring.devs[0]['errors'],
                              self.app.error_suppression_limit + 1)
            self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200),
                                   503)
            self.assert_('last_error' in controller.app.object_ring.devs[0])
            self.assert_status_map(controller.PUT, (200, 200, 200, 201, 201,
                                                    201), 503)
            self.assert_status_map(controller.POST,
                                   (200, 200, 200, 200, 200, 200, 202, 202,
                                    202), 503)
            self.assert_status_map(controller.DELETE,
                                   (200, 200, 200, 204, 204, 204), 503)
            self.app.error_suppression_interval = -300
            self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200),
                                   200)
            self.assertRaises(BaseException,
                              self.assert_status_map, controller.DELETE,
                              (200, 200, 200, 204, 204, 204), 503,
                              raise_exc=True)

    def test_acc_or_con_missing_returns_404(self):
        with save_globals():
            self.app.memcache = FakeMemcacheReturnsNone()
            for dev in self.app.account_ring.devs.values():
                del dev['errors']
                del dev['last_error']
            for dev in self.app.container_ring.devs.values():
                del dev['errors']
                del dev['last_error']
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 200, 200, 200, 200)
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
            self.app.update_request(req)
            resp = getattr(controller, 'DELETE')(req)
            self.assertEquals(resp.status_int, 200)

            set_http_connect(404, 404, 404)
            #                acct acct acct
            resp = getattr(controller, 'DELETE')(req)
            self.assertEquals(resp.status_int, 404)

            set_http_connect(503, 404, 404)
            #                acct acct acct
            resp = getattr(controller, 'DELETE')(req)
            self.assertEquals(resp.status_int, 404)

            set_http_connect(503, 503, 404)
            #                acct acct acct
            resp = getattr(controller, 'DELETE')(req)
            self.assertEquals(resp.status_int, 404)

            set_http_connect(503, 503, 503)
            #                acct acct acct
            resp = getattr(controller, 'DELETE')(req)
            self.assertEquals(resp.status_int, 404)

            set_http_connect(200, 200, 204, 204, 204)
            #                acct cont obj  obj  obj
            resp = getattr(controller, 'DELETE')(req)
            self.assertEquals(resp.status_int, 204)

            set_http_connect(200, 404, 404, 404)
            #                acct cont cont cont
            resp = getattr(controller, 'DELETE')(req)
            self.assertEquals(resp.status_int, 404)

            set_http_connect(200, 503, 503, 503)
            #                acct cont cont cont
            resp = getattr(controller, 'DELETE')(req)
            self.assertEquals(resp.status_int, 404)

            for dev in self.app.account_ring.devs.values():
                dev['errors'] = self.app.error_suppression_limit + 1
                dev['last_error'] = time.time()
            set_http_connect(200)
            #                acct [isn't actually called since everything
            #                      is error limited]
            resp = getattr(controller, 'DELETE')(req)
            self.assertEquals(resp.status_int, 404)

            for dev in self.app.account_ring.devs.values():
                dev['errors'] = 0
            for dev in self.app.container_ring.devs.values():
                dev['errors'] = self.app.error_suppression_limit + 1
                dev['last_error'] = time.time()
            set_http_connect(200, 200)
            #                acct cont [isn't actually called since
            #                           everything is error limited]
            resp = getattr(controller, 'DELETE')(req)
            self.assertEquals(resp.status_int, 404)

    def test_PUT_POST_requires_container_exist(self):
        with save_globals():
            self.app.object_post_as_copy = False
            self.app.memcache = FakeMemcacheReturnsNone()
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')

            set_http_connect(200, 404, 404, 404, 200, 200, 200)
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'})
            self.app.update_request(req)
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 404)

            set_http_connect(200, 404, 404, 404, 200, 200)
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
                                headers={'Content-Type': 'text/plain'})
            self.app.update_request(req)
            resp = controller.POST(req)
            self.assertEquals(resp.status_int, 404)

    def test_PUT_POST_as_copy_requires_container_exist(self):
        with save_globals():
            self.app.memcache = FakeMemcacheReturnsNone()
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 404, 404, 404, 200, 200, 200)
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'})
            self.app.update_request(req)
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 404)

            set_http_connect(200, 404, 404, 404, 200, 200, 200, 200, 200, 200)
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
                                headers={'Content-Type': 'text/plain'})
            self.app.update_request(req)
            resp = controller.POST(req)
            self.assertEquals(resp.status_int, 404)

    def test_bad_metadata(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 201, 201, 201)
            #                acct cont obj  obj  obj
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0'})
            self.app.update_request(req)
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)

            set_http_connect(201, 201, 201)
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0',
                                         'X-Object-Meta-' + ('a' *
                                         MAX_META_NAME_LENGTH): 'v'})
            self.app.update_request(req)
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)
            set_http_connect(201, 201, 201)
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0',
                                         'X-Object-Meta-' + ('a' *
                                         (MAX_META_NAME_LENGTH + 1)): 'v'})
            self.app.update_request(req)
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 400)

            set_http_connect(201, 201, 201)
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0',
                                         'X-Object-Meta-Too-Long': 'a' *
                                         MAX_META_VALUE_LENGTH})
            self.app.update_request(req)
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)
            set_http_connect(201, 201, 201)
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0',
                                         'X-Object-Meta-Too-Long': 'a' *
                                         (MAX_META_VALUE_LENGTH + 1)})
            self.app.update_request(req)
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 400)

            set_http_connect(201, 201, 201)
            headers = {'Content-Length': '0'}
            for x in xrange(MAX_META_COUNT):
                headers['X-Object-Meta-%d' % x] = 'v'
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers=headers)
            self.app.update_request(req)
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)
            set_http_connect(201, 201, 201)
            headers = {'Content-Length': '0'}
            for x in xrange(MAX_META_COUNT + 1):
                headers['X-Object-Meta-%d' % x] = 'v'
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers=headers)
            self.app.update_request(req)
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 400)

            set_http_connect(201, 201, 201)
            headers = {'Content-Length': '0'}
            header_value = 'a' * MAX_META_VALUE_LENGTH
            size = 0
            x = 0
            while size < MAX_META_OVERALL_SIZE - 4 - \
                    MAX_META_VALUE_LENGTH:
                size += 4 + MAX_META_VALUE_LENGTH
                headers['X-Object-Meta-%04d' % x] = header_value
                x += 1
            if MAX_META_OVERALL_SIZE - size > 1:
                headers['X-Object-Meta-a'] = \
                    'a' * (MAX_META_OVERALL_SIZE - size - 1)
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers=headers)
            self.app.update_request(req)
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)
            set_http_connect(201, 201, 201)
            headers['X-Object-Meta-a'] = \
                'a' * (MAX_META_OVERALL_SIZE - size)
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers=headers)
            self.app.update_request(req)
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 400)

    def test_copy_from(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            # initial source object PUT
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0'})
            self.app.update_request(req)
            set_http_connect(200, 200, 201, 201, 201)
            #                acct cont obj  obj  obj
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)

            # basic copy
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0',
                                         'X-Copy-From': 'c/o'})
            self.app.update_request(req)
            set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201)
            #                acct cont acct cont objc objc objc obj  obj  obj
            self.app.memcache.store = {}
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)
            self.assertEquals(resp.headers['x-copied-from'], 'c/o')

            # non-zero content length
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '5',
                                         'X-Copy-From': 'c/o'})
            self.app.update_request(req)
            set_http_connect(200, 200, 200, 200, 200, 200, 200)
            #                acct cont acct cont objc objc objc
            self.app.memcache.store = {}
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 400)

            # extra source path parsing
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0',
                                         'X-Copy-From': 'c/o/o2'})
            req.account = 'a'
            set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201)
            #                acct cont acct cont objc objc objc obj  obj  obj
            self.app.memcache.store = {}
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)
            self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2')

            # space in soure path
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0',
                                         'X-Copy-From': 'c/o%20o2'})
            req.account = 'a'
            set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201)
            #                acct cont acct cont objc objc objc obj  obj  obj
            self.app.memcache.store = {}
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)
            self.assertEquals(resp.headers['x-copied-from'], 'c/o%20o2')

            # repeat tests with leading /
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0',
                                         'X-Copy-From': '/c/o'})
            self.app.update_request(req)
            set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201)
            #                acct cont acct cont objc objc objc obj  obj  obj
            self.app.memcache.store = {}
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)
            self.assertEquals(resp.headers['x-copied-from'], 'c/o')

            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0',
                                         'X-Copy-From': '/c/o/o2'})
            req.account = 'a'
            set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201)
            #                acct cont acct cont objc objc objc obj  obj  obj
            self.app.memcache.store = {}
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)
            self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2')

            # negative tests

            # invalid x-copy-from path
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0',
                                         'X-Copy-From': '/c'})
            self.app.update_request(req)
            self.app.memcache.store = {}
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int // 100, 4)  # client error

            # server error
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0',
                                         'X-Copy-From': '/c/o'})
            self.app.update_request(req)
            set_http_connect(200, 200, 503, 503, 503)
            #                acct cont objc objc objc
            self.app.memcache.store = {}
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 503)

            # not found
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0',
                                         'X-Copy-From': '/c/o'})
            self.app.update_request(req)
            set_http_connect(200, 200, 404, 404, 404)
            #                acct cont objc objc objc
            self.app.memcache.store = {}
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 404)

            # some missing containers
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0',
                                         'X-Copy-From': '/c/o'})
            self.app.update_request(req)
            set_http_connect(200, 200, 404, 404, 200, 201, 201, 201)
            #                acct cont objc objc objc obj  obj  obj
            self.app.memcache.store = {}
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)

            # test object meta data
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0',
                                         'X-Copy-From': '/c/o',
                                         'X-Object-Meta-Ours': 'okay'})
            self.app.update_request(req)
            set_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
            #                acct cont objc objc objc obj  obj  obj
            self.app.memcache.store = {}
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)
            self.assertEquals(resp.headers.get('x-object-meta-test'),
                              'testing')
            self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay')

            # copy-from object is too large to fit in target object
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0',
                                         'X-Copy-From': '/c/o'})
            self.app.update_request(req)

            class LargeResponseBody(object):

                def __len__(self):
                    return MAX_FILE_SIZE + 1

                def __getitem__(self, key):
                    return ''

            copy_from_obj_body = LargeResponseBody()
            set_http_connect(200, 200, 200, 200, 200, 201, 201, 201,
                body=copy_from_obj_body)
            self.app.memcache.store = {}
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 413)

    def test_COPY(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0'})
            req.account = 'a'
            set_http_connect(200, 200, 201, 201, 201)
            #                acct cont obj  obj  obj
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 201)

            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
                                headers={'Destination': 'c/o'})
            req.account = 'a'
            set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201)
            #                acct cont acct cont objc objc objc obj  obj  obj
            self.app.memcache.store = {}
            resp = controller.COPY(req)
            self.assertEquals(resp.status_int, 201)
            self.assertEquals(resp.headers['x-copied-from'], 'c/o')

            req = Request.blank('/a/c/o/o2',
                                environ={'REQUEST_METHOD': 'COPY'},
                                headers={'Destination': 'c/o'})
            req.account = 'a'
            controller.object_name = 'o/o2'
            set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201)
            #                acct cont acct cont objc objc objc obj  obj  obj
            self.app.memcache.store = {}
            resp = controller.COPY(req)
            self.assertEquals(resp.status_int, 201)
            self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2')

            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
                                headers={'Destination': '/c/o'})
            req.account = 'a'
            controller.object_name = 'o'
            set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201)
            #                acct cont acct cont objc objc objc obj  obj  obj
            self.app.memcache.store = {}
            resp = controller.COPY(req)
            self.assertEquals(resp.status_int, 201)
            self.assertEquals(resp.headers['x-copied-from'], 'c/o')

            req = Request.blank('/a/c/o/o2',
                                environ={'REQUEST_METHOD': 'COPY'},
                                headers={'Destination': '/c/o'})
            req.account = 'a'
            controller.object_name = 'o/o2'
            set_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, 201)
            #                acct cont acct cont objc objc objc obj  obj  obj
            self.app.memcache.store = {}
            resp = controller.COPY(req)
            self.assertEquals(resp.status_int, 201)
            self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2')

            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
                                headers={'Destination': 'c_o'})
            req.account = 'a'
            controller.object_name = 'o'
            set_http_connect(200, 200)
            #                acct cont
            self.app.memcache.store = {}
            resp = controller.COPY(req)
            self.assertEquals(resp.status_int, 412)

            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
                                headers={'Destination': '/c/o'})
            req.account = 'a'
            controller.object_name = 'o'
            set_http_connect(200, 200, 503, 503, 503)
            #                acct cont objc objc objc
            self.app.memcache.store = {}
            resp = controller.COPY(req)
            self.assertEquals(resp.status_int, 503)

            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
                                headers={'Destination': '/c/o'})
            req.account = 'a'
            controller.object_name = 'o'
            set_http_connect(200, 200, 404, 404, 404)
            #                acct cont objc objc objc
            self.app.memcache.store = {}
            resp = controller.COPY(req)
            self.assertEquals(resp.status_int, 404)

            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
                                headers={'Destination': '/c/o'})
            req.account = 'a'
            controller.object_name = 'o'
            set_http_connect(200, 200, 404, 404, 200, 201, 201, 201)
            #                acct cont objc objc objc obj  obj  obj
            self.app.memcache.store = {}
            resp = controller.COPY(req)
            self.assertEquals(resp.status_int, 201)

            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
                                headers={'Destination': '/c/o',
                                         'X-Object-Meta-Ours': 'okay'})
            req.account = 'a'
            controller.object_name = 'o'
            set_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
            #                acct cont objc objc objc obj  obj  obj
            self.app.memcache.store = {}
            resp = controller.COPY(req)
            self.assertEquals(resp.status_int, 201)
            self.assertEquals(resp.headers.get('x-object-meta-test'),
                              'testing')
            self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay')

            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
                                headers={'Destination': '/c/o'})
            self.app.update_request(req)

            class LargeResponseBody(object):

                def __len__(self):
                    return MAX_FILE_SIZE + 1

                def __getitem__(self, key):
                    return ''

            copy_from_obj_body = LargeResponseBody()
            set_http_connect(200, 200, 200, 200, 200, 201, 201, 201,
                body=copy_from_obj_body)
            self.app.memcache.store = {}
            resp = controller.COPY(req)
            self.assertEquals(resp.status_int, 413)

    def test_COPY_newest(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
                                headers={'Destination': '/c/o'})
            req.account = 'a'
            controller.object_name = 'o'
            set_http_connect(200, 200, 200, 200, 200, 201, 201, 201,
                             #act cont objc objc objc obj  obj  obj
                             timestamps=('1', '1', '1', '3', '2', '4', '4',
                                         '4'))
            self.app.memcache.store = {}
            resp = controller.COPY(req)
            self.assertEquals(resp.status_int, 201)
            self.assertEquals(resp.headers['x-copied-from-last-modified'],
                              '3')

    def test_chunked_put(self):

        class ChunkedFile():

            def __init__(self, bytes):
                self.bytes = bytes
                self.read_bytes = 0

            @property
            def bytes_left(self):
                return self.bytes - self.read_bytes

            def read(self, amt=None):
                if self.read_bytes >= self.bytes:
                    raise StopIteration()
                if not amt:
                    amt = self.bytes_left
                data = 'a' * min(amt, self.bytes_left)
                self.read_bytes += len(data)
                return data

        with save_globals():
            set_http_connect(201, 201, 201, 201)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
                                headers={'Transfer-Encoding': 'chunked',
                                         'Content-Type': 'foo/bar'})

            req.body_file = ChunkedFile(10)
            self.app.memcache.store = {}
            self.app.update_request(req)
            res = controller.PUT(req)
            self.assertEquals(res.status_int // 100, 2)  # success

            # test 413 entity to large
            set_http_connect(201, 201, 201, 201)
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
                                headers={'Transfer-Encoding': 'chunked',
                                'Content-Type': 'foo/bar'})
            req.body_file = ChunkedFile(11)
            self.app.memcache.store = {}
            self.app.update_request(req)
            try:
                swift.proxy.controllers.obj.MAX_FILE_SIZE = 10
                res = controller.PUT(req)
                self.assertEquals(res.status_int, 413)
            finally:
                swift.proxy.controllers.obj.MAX_FILE_SIZE = MAX_FILE_SIZE

    def test_chunked_put_bad_version(self):
        # Check bad version
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
         obj2lis) = _test_sockets
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v0 HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nContent-Length: 0\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 412'
        self.assertEquals(headers[:len(exp)], exp)

    def test_chunked_put_bad_path(self):
        # Check bad path
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
         obj2lis) = _test_sockets
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET invalid HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nContent-Length: 0\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 404'
        self.assertEquals(headers[:len(exp)], exp)

    def test_chunked_put_bad_utf8(self):
        # Check invalid utf-8
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
         obj2lis) = _test_sockets
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a%80 HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Auth-Token: t\r\n'
                 'Content-Length: 0\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 412'
        self.assertEquals(headers[:len(exp)], exp)

    def test_chunked_put_bad_path_no_controller(self):
        # Check bad path, no controller
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
         obj2lis) = _test_sockets
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1 HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Auth-Token: t\r\n'
                 'Content-Length: 0\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 412'
        self.assertEquals(headers[:len(exp)], exp)

    def test_chunked_put_bad_method(self):
        # Check bad method
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
         obj2lis) = _test_sockets
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('LICK /v1/a HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Auth-Token: t\r\n'
                 'Content-Length: 0\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 405'
        self.assertEquals(headers[:len(exp)], exp)

    def test_chunked_put_unhandled_exception(self):
        # Check unhandled exception
        (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv,
         obj2srv) = _test_servers
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
         obj2lis) = _test_sockets
        orig_update_request = prosrv.update_request

        def broken_update_request(*args, **kwargs):
            raise Exception('fake: this should be printed')

        prosrv.update_request = broken_update_request
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('HEAD /v1/a HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Auth-Token: t\r\n'
                 'Content-Length: 0\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 500'
        self.assertEquals(headers[:len(exp)], exp)
        prosrv.update_request = orig_update_request

    def test_chunked_put_head_account(self):
        # Head account, just a double check and really is here to test
        # the part Application.log_request that 'enforces' a
        # content_length on the response.
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
         obj2lis) = _test_sockets
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('HEAD /v1/a HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Auth-Token: t\r\n'
                 'Content-Length: 0\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 204'
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('\r\nContent-Length: 0\r\n' in headers)

    def test_chunked_put_utf8_all_the_way_down(self):
        # Test UTF-8 Unicode all the way through the system
        ustr = '\xe1\xbc\xb8\xce\xbf\xe1\xbd\xba \xe1\xbc\xb0\xce' \
               '\xbf\xe1\xbd\xbb\xce\x87 \xcf\x84\xe1\xbd\xb0 \xcf' \
               '\x80\xe1\xbd\xb1\xce\xbd\xcf\x84\xca\xbc \xe1\xbc' \
               '\x82\xce\xbd \xe1\xbc\x90\xce\xbe\xe1\xbd\xb5\xce' \
               '\xba\xce\xbf\xce\xb9 \xcf\x83\xce\xb1\xcf\x86\xe1' \
               '\xbf\x86.Test'
        ustr_short = '\xe1\xbc\xb8\xce\xbf\xe1\xbd\xbatest'
        # Create ustr container
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
         obj2lis) = _test_sockets
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Storage-Token: t\r\n'
                 'Content-Length: 0\r\n\r\n' % quote(ustr))
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # List account with ustr container (test plain)
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Storage-Token: t\r\n'
                 'Content-Length: 0\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        containers = fd.read().split('\n')
        self.assert_(ustr in containers)
        # List account with ustr container (test json)
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a?format=json HTTP/1.1\r\n'
                 'Host: localhost\r\nConnection: close\r\n'
                 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        listing = simplejson.loads(fd.read())
        self.assert_(ustr.decode('utf8') in [l['name'] for l in listing])
        # List account with ustr container (test xml)
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a?format=xml HTTP/1.1\r\n'
                 'Host: localhost\r\nConnection: close\r\n'
                 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('<name>%s</name>' % ustr in fd.read())
        # Create ustr object with ustr metadata in ustr container
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Storage-Token: t\r\n'
                 'X-Object-Meta-%s: %s\r\nContent-Length: 0\r\n\r\n' %
                 (quote(ustr), quote(ustr), quote(ustr_short),
                  quote(ustr)))
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # List ustr container with ustr object (test plain)
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Storage-Token: t\r\n'
                 'Content-Length: 0\r\n\r\n' % quote(ustr))
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        objects = fd.read().split('\n')
        self.assert_(ustr in objects)
        # List ustr container with ustr object (test json)
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/%s?format=json HTTP/1.1\r\n'
                 'Host: localhost\r\nConnection: close\r\n'
                 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n' %
                 quote(ustr))
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        listing = simplejson.loads(fd.read())
        self.assertEquals(listing[0]['name'], ustr.decode('utf8'))
        # List ustr container with ustr object (test xml)
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/%s?format=xml HTTP/1.1\r\n'
                 'Host: localhost\r\nConnection: close\r\n'
                 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n' %
                 quote(ustr))
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('<name>%s</name>' % ustr in fd.read())
        # Retrieve ustr object with ustr metadata
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Storage-Token: t\r\n'
                 'Content-Length: 0\r\n\r\n' %
                 (quote(ustr), quote(ustr)))
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('\r\nX-Object-Meta-%s: %s\r\n' %
                     (quote(ustr_short).lower(), quote(ustr)) in headers)

    def test_chunked_put_chunked_put(self):
        # Do chunked object put
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
         obj2lis) = _test_sockets
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        # Also happens to assert that x-storage-token is taken as a
        # replacement for x-auth-token.
        fd.write('PUT /v1/a/c/o/chunky HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Storage-Token: t\r\n'
                 'Transfer-Encoding: chunked\r\n\r\n'
                 '2\r\noh\r\n4\r\n hai\r\nf\r\n123456789abcdef\r\n'
                 '0\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # Ensure we get what we put
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/c/o/chunky HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Auth-Token: t\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        body = fd.read()
        self.assertEquals(body, 'oh hai123456789abcdef')

    def test_version_manifest(self):
        versions_to_create = 3
        # Create a container for our versioned object testing
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
         obj2lis) = _test_sockets
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/versions HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Storage-Token: t\r\n'
                 'Content-Length: 0\r\nX-Versions-Location: vers\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # check that the header was set
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/versions HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Storage-Token: t\r\n\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 2'  # 2xx series response
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('X-Versions-Location: vers' in headers)
        # make the container for the object versions
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/vers HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Storage-Token: t\r\n'
                 'Content-Length: 0\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # Create the versioned file
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/versions/name HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Storage-Token: '
                 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish0\r\n'
                 'X-Object-Meta-Foo: barbaz\r\n\r\n00000\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # Create the object versions
        for segment in xrange(1, versions_to_create):
            sleep(.01)  # guarantee that the timestamp changes
            sock = connect_tcp(('localhost', prolis.getsockname()[1]))
            fd = sock.makefile()
            fd.write('PUT /v1/a/versions/name HTTP/1.1\r\nHost: '
                     'localhost\r\nConnection: close\r\nX-Storage-Token: '
                     't\r\nContent-Length: 5\r\nContent-Type: text/jibberish%s'
                     '\r\n\r\n%05d\r\n' % (segment, segment))
            fd.flush()
            headers = readuntil2crlfs(fd)
            exp = 'HTTP/1.1 201'
            self.assertEquals(headers[:len(exp)], exp)
            # Ensure retrieving the manifest file gets the latest version
            sock = connect_tcp(('localhost', prolis.getsockname()[1]))
            fd = sock.makefile()
            fd.write('GET /v1/a/versions/name HTTP/1.1\r\nHost: '
                     'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n'
                     '\r\n')
            fd.flush()
            headers = readuntil2crlfs(fd)
            exp = 'HTTP/1.1 200'
            self.assertEquals(headers[:len(exp)], exp)
            self.assert_('Content-Type: text/jibberish%s' % segment in headers)
            self.assert_('X-Object-Meta-Foo: barbaz' not in headers)
            body = fd.read()
            self.assertEquals(body, '%05d' % segment)
        # Ensure we have the right number of versions saved
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/vers?prefix=004name/ HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        body = fd.read()
        versions = [x for x in body.split('\n') if x]
        self.assertEquals(len(versions), versions_to_create - 1)
        # copy a version and make sure the version info is stripped
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('COPY /v1/a/versions/name HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Auth-Token: '
                 't\r\nDestination: versions/copied_name\r\n'
                 'Content-Length: 0\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 2'  # 2xx series response to the COPY
        self.assertEquals(headers[:len(exp)], exp)
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/versions/copied_name HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        body = fd.read()
        self.assertEquals(body, '%05d' % segment)
        # post and make sure it's updated
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('POST /v1/a/versions/name HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Auth-Token: '
                 't\r\nContent-Type: foo/bar\r\nContent-Length: 0\r\n'
                 'X-Object-Meta-Bar: foo\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 2'  # 2xx series response to the POST
        self.assertEquals(headers[:len(exp)], exp)
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/versions/name HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('Content-Type: foo/bar' in headers)
        self.assert_('X-Object-Meta-Bar: foo' in headers)
        body = fd.read()
        self.assertEquals(body, '%05d' % segment)
        # Delete the object versions
        for segment in xrange(versions_to_create - 1, 0, -1):
            sock = connect_tcp(('localhost', prolis.getsockname()[1]))
            fd = sock.makefile()
            fd.write('DELETE /v1/a/versions/name HTTP/1.1\r\nHost: localhost\r'
                     '\nConnection: close\r\nX-Storage-Token: t\r\n\r\n')
            fd.flush()
            headers = readuntil2crlfs(fd)
            exp = 'HTTP/1.1 2'  # 2xx series response
            self.assertEquals(headers[:len(exp)], exp)
            # Ensure retrieving the manifest file gets the latest version
            sock = connect_tcp(('localhost', prolis.getsockname()[1]))
            fd = sock.makefile()
            fd.write('GET /v1/a/versions/name HTTP/1.1\r\nHost: localhost\r\n'
                     'Connection: close\r\nX-Auth-Token: t\r\n\r\n')
            fd.flush()
            headers = readuntil2crlfs(fd)
            exp = 'HTTP/1.1 200'
            self.assertEquals(headers[:len(exp)], exp)
            self.assert_('Content-Type: text/jibberish%s' % (segment - 1)
                         in headers)
            body = fd.read()
            self.assertEquals(body, '%05d' % (segment - 1))
            # Ensure we have the right number of versions saved
            sock = connect_tcp(('localhost', prolis.getsockname()[1]))
            fd = sock.makefile()
            fd.write('GET /v1/a/vers?prefix=004name/ HTTP/1.1\r\nHost: '
                     'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r'
                     '\n')
            fd.flush()
            headers = readuntil2crlfs(fd)
            exp = 'HTTP/1.1 2'  # 2xx series response
            self.assertEquals(headers[:len(exp)], exp)
            body = fd.read()
            versions = [x for x in body.split('\n') if x]
            self.assertEquals(len(versions), segment - 1)
        # there is now one segment left (in the manifest)
        # Ensure we have no saved versions
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/vers?prefix=004name/ HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 204 No Content'
        self.assertEquals(headers[:len(exp)], exp)
        # delete the last verision
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('DELETE /v1/a/versions/name HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Storage-Token: t\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 2'  # 2xx series response
        self.assertEquals(headers[:len(exp)], exp)
        # Ensure it's all gone
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/versions/name HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 404'
        self.assertEquals(headers[:len(exp)], exp)

        # make sure manifest files don't get versioned
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/versions/name HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Storage-Token: '
                 't\r\nContent-Length: 0\r\nContent-Type: text/jibberish0\r\n'
                 'Foo: barbaz\r\nX-Object-Manifest: vers/foo_\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # Ensure we have no saved versions
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/vers?prefix=004name/ HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 204 No Content'
        self.assertEquals(headers[:len(exp)], exp)

        # DELETE v1/a/c/obj shouldn't delete v1/a/c/obj/sub versions
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/versions/name HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Storage-Token: '
                 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish0\r\n'
                 'Foo: barbaz\r\n\r\n00000\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/versions/name HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Storage-Token: '
                 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish0\r\n'
                 'Foo: barbaz\r\n\r\n00001\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/versions/name/sub HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Storage-Token: '
                 't\r\nContent-Length: 4\r\nContent-Type: text/jibberish0\r\n'
                 'Foo: barbaz\r\n\r\nsub1\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/versions/name/sub HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Storage-Token: '
                 't\r\nContent-Length: 4\r\nContent-Type: text/jibberish0\r\n'
                 'Foo: barbaz\r\n\r\nsub2\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('DELETE /v1/a/versions/name HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Storage-Token: t\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 2'  # 2xx series response
        self.assertEquals(headers[:len(exp)], exp)
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/vers?prefix=008name/sub/ HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 2'  # 2xx series response
        self.assertEquals(headers[:len(exp)], exp)
        body = fd.read()
        versions = [x for x in body.split('\n') if x]
        self.assertEquals(len(versions), 1)

        # Check for when the versions target container doesn't exist
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/whoops HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Storage-Token: t\r\n'
                 'Content-Length: 0\r\nX-Versions-Location: none\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # Create the versioned file
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/whoops/foo HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Storage-Token: '
                 't\r\nContent-Length: 5\r\n\r\n00000\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # Create another version
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/whoops/foo HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Storage-Token: '
                 't\r\nContent-Length: 5\r\n\r\n00001\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 412'
        self.assertEquals(headers[:len(exp)], exp)
        # Delete the object
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('DELETE /v1/a/whoops/foo HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Storage-Token: t\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 2'  # 2xx response
        self.assertEquals(headers[:len(exp)], exp)

    def test_chunked_put_lobjects_with_nonzero_size_manifest_file(self):
        # Create a container for our segmented/manifest object testing
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \
            _test_sockets
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/segmented_nonzero HTTP/1.1\r\nHost: localhost\r\n'
                 'Connection: close\r\nX-Storage-Token: t\r\n'
                 'Content-Length: 0\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # Create the object segments
        segment_etags = []
        for segment in xrange(5):
            sock = connect_tcp(('localhost', prolis.getsockname()[1]))
            fd = sock.makefile()
            fd.write('PUT /v1/a/segmented_nonzero/name/%s HTTP/1.1\r\nHost: '
                     'localhost\r\nConnection: close\r\nX-Storage-Token: '
                     't\r\nContent-Length: 5\r\n\r\n1234 ' % str(segment))
            fd.flush()
            headers = readuntil2crlfs(fd)
            exp = 'HTTP/1.1 201'
            self.assertEquals(headers[:len(exp)], exp)
            segment_etags.append(md5('1234 ').hexdigest())

        # Create the nonzero size manifest file
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Storage-Token: '
                 't\r\nContent-Length: 5\r\n\r\nabcd ')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)

        # Create the object manifest file
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('POST /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Storage-Token: t\r\n'
                 'X-Object-Manifest: segmented_nonzero/name/\r\n'
                 'Foo: barbaz\r\nContent-Type: text/jibberish\r\n'
                 '\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 202'
        self.assertEquals(headers[:len(exp)], exp)

        # Ensure retrieving the manifest file gets the whole object
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Auth-Token: '
                 't\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers)
        self.assert_('Content-Type: text/jibberish' in headers)
        self.assert_('Foo: barbaz' in headers)
        expected_etag = md5(''.join(segment_etags)).hexdigest()
        self.assert_('Etag: "%s"' % expected_etag in headers)
        body = fd.read()
        self.assertEquals(body, '1234 1234 1234 1234 1234 ')

        # Get lobjects with Range smaller than manifest file
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n'
                 'Range: bytes=0-4\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 206'
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers)
        self.assert_('Content-Type: text/jibberish' in headers)
        self.assert_('Foo: barbaz' in headers)
        expected_etag = md5(''.join(segment_etags)).hexdigest()
        body = fd.read()
        self.assertEquals(body, '1234 ')

        # Get lobjects with Range bigger than manifest file
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
                 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n'
                 'Range: bytes=11-15\r\n\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 206'
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers)
        self.assert_('Content-Type: text/jibberish' in headers)
        self.assert_('Foo: barbaz' in headers)
        expected_etag = md5(''.join(segment_etags)).hexdigest()
        body = fd.read()
        self.assertEquals(body, '234 1')

    def test_chunked_put_lobjects(self):
        # Create a container for our segmented/manifest object testing
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
         obj2lis) = _test_sockets
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/segmented%20object HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Storage-Token: t\r\n'
                 'Content-Length: 0\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # Create the object segments
        segment_etags = []
        for segment in xrange(5):
            sock = connect_tcp(('localhost', prolis.getsockname()[1]))
            fd = sock.makefile()
            fd.write('PUT /v1/a/segmented%%20object/object%%20name/%s '
                     'HTTP/1.1\r\n'
                     'Host: localhost\r\n'
                     'Connection: close\r\n'
                     'X-Storage-Token: t\r\n'
                     'Content-Length: 5\r\n'
                     '\r\n'
                     '1234 ' % str(segment))
            fd.flush()
            headers = readuntil2crlfs(fd)
            exp = 'HTTP/1.1 201'
            self.assertEquals(headers[:len(exp)], exp)
            segment_etags.append(md5('1234 ').hexdigest())
        # Create the object manifest file
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/segmented%20object/object%20name HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Storage-Token: t\r\n'
                 'Content-Length: 0\r\n'
                 'X-Object-Manifest: segmented%20object/object%20name/\r\n'
                 'Content-Type: text/jibberish\r\n'
                 'Foo: barbaz\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # Check retrieving the listing the manifest would retrieve
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/segmented%20object?prefix=object%20name/ '
                 'HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Auth-Token: t\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        body = fd.read()
        self.assertEquals(
            body,
            'object name/0\n'
            'object name/1\n'
            'object name/2\n'
            'object name/3\n'
            'object name/4\n')
        # Ensure retrieving the manifest file gets the whole object
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/segmented%20object/object%20name HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Auth-Token: t\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('X-Object-Manifest: segmented%20object/object%20name/' in
                     headers)
        self.assert_('Content-Type: text/jibberish' in headers)
        self.assert_('Foo: barbaz' in headers)
        expected_etag = md5(''.join(segment_etags)).hexdigest()
        self.assert_('Etag: "%s"' % expected_etag in headers)
        body = fd.read()
        self.assertEquals(body, '1234 1234 1234 1234 1234 ')
        # Do it again but exceeding the container listing limit
        swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = 2
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))

        fd = sock.makefile()
        fd.write('GET /v1/a/segmented%20object/object%20name HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Auth-Token: t\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('X-Object-Manifest: segmented%20object/object%20name/' in
                     headers)
        self.assert_('Content-Type: text/jibberish' in headers)
        body = fd.read()
        # A bit fragile of a test; as it makes the assumption that all
        # will be sent in a single chunk.
        self.assertEquals(
            body, '19\r\n1234 1234 1234 1234 1234 \r\n0\r\n\r\n')
        # Make a copy of the manifested object, which should
        # error since the number of segments exceeds
        # CONTAINER_LISTING_LIMIT.
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/segmented%20object/copy HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Auth-Token: t\r\n'
                 'X-Copy-From: segmented%20object/object%20name\r\n'
                 'Content-Length: 0\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 413'
        self.assertEquals(headers[:len(exp)], exp)
        body = fd.read()
        # After adjusting the CONTAINER_LISTING_LIMIT, make a copy of
        # the manifested object which should consolidate the segments.
        swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = 10000
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/segmented%20object/copy HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Auth-Token: t\r\n'
                 'X-Copy-From: segmented%20object/object%20name\r\n'
                 'Content-Length: 0\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        body = fd.read()
        # Retrieve and validate the copy.
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/segmented%20object/copy HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Auth-Token: t\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('x-object-manifest:' not in headers.lower())
        self.assert_('Content-Length: 25\r' in headers)
        body = fd.read()
        self.assertEquals(body, '1234 1234 1234 1234 1234 ')
        # Create an object manifest file pointing to nothing
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/segmented%20object/empty HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Storage-Token: t\r\n'
                 'Content-Length: 0\r\n'
                 'X-Object-Manifest: segmented%20object/empty/\r\n'
                 'Content-Type: text/jibberish\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # Ensure retrieving the manifest file gives a zero-byte file
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/segmented%20object/empty HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Auth-Token: t\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('X-Object-Manifest: segmented%20object/empty/' in headers)
        self.assert_('Content-Type: text/jibberish' in headers)
        body = fd.read()
        self.assertEquals(body, '')
        # Check copy content type
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/c/obj HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Storage-Token: t\r\n'
                 'Content-Length: 0\r\n'
                 'Content-Type: text/jibberish\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/c/obj2 HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Storage-Token: t\r\n'
                 'Content-Length: 0\r\n'
                 'X-Copy-From: c/obj\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # Ensure getting the copied file gets original content-type
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/c/obj2 HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Auth-Token: t\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('Content-Type: text/jibberish' in headers)
        # Check set content type
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/c/obj3 HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Storage-Token: t\r\n'
                 'Content-Length: 0\r\n'
                 'Content-Type: foo/bar\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # Ensure getting the copied file gets original content-type
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/c/obj3 HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Auth-Token: t\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('Content-Type: foo/bar' in
                     headers.split('\r\n'), repr(headers.split('\r\n')))
        # Check set content type with charset
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/c/obj4 HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Storage-Token: t\r\n'
                 'Content-Length: 0\r\n'
                 'Content-Type: foo/bar; charset=UTF-8\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEquals(headers[:len(exp)], exp)
        # Ensure getting the copied file gets original content-type
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/c/obj4 HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Auth-Token: t\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEquals(headers[:len(exp)], exp)
        self.assert_('Content-Type: foo/bar; charset=UTF-8' in
                     headers.split('\r\n'), repr(headers.split('\r\n')))

    def test_mismatched_etags(self):
        with save_globals():
            # no etag supplied, object servers return success w/ diff values
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '0'})
            self.app.update_request(req)
            set_http_connect(200, 201, 201, 201,
                             etags=[None,
                                    '68b329da9893e34099c7d8ad5cb9c940',
                                    '68b329da9893e34099c7d8ad5cb9c940',
                                    '68b329da9893e34099c7d8ad5cb9c941'])
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int // 100, 5)  # server error

            # req supplies etag, object servers return 422 - mismatch
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={
                                    'Content-Length': '0',
                                    'ETag': '68b329da9893e34099c7d8ad5cb9c940',
                                })
            self.app.update_request(req)
            set_http_connect(200, 422, 422, 503,
                             etags=['68b329da9893e34099c7d8ad5cb9c940',
                                    '68b329da9893e34099c7d8ad5cb9c941',
                                    None,
                                    None])
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int // 100, 4)  # client error

    def test_response_get_accept_ranges_header(self):
        with save_globals():
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'})
            self.app.update_request(req)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 200)
            resp = controller.GET(req)
            self.assert_('accept-ranges' in resp.headers)
            self.assertEquals(resp.headers['accept-ranges'], 'bytes')

    def test_response_head_accept_ranges_header(self):
        with save_globals():
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'HEAD'})
            self.app.update_request(req)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 200)
            resp = controller.HEAD(req)
            self.assert_('accept-ranges' in resp.headers)
            self.assertEquals(resp.headers['accept-ranges'], 'bytes')

    def test_GET_calls_authorize(self):
        called = [False]

        def authorize(req):
            called[0] = True
            return HTTPUnauthorized(request=req)
        with save_globals():
            set_http_connect(200, 200, 201, 201, 201)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            req = Request.blank('/a/c/o')
            req.environ['swift.authorize'] = authorize
            self.app.update_request(req)
            res = controller.GET(req)
        self.assert_(called[0])

    def test_HEAD_calls_authorize(self):
        called = [False]

        def authorize(req):
            called[0] = True
            return HTTPUnauthorized(request=req)
        with save_globals():
            set_http_connect(200, 200, 201, 201, 201)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            req = Request.blank('/a/c/o', {'REQUEST_METHOD': 'HEAD'})
            req.environ['swift.authorize'] = authorize
            self.app.update_request(req)
            res = controller.HEAD(req)
        self.assert_(called[0])

    def test_POST_calls_authorize(self):
        called = [False]

        def authorize(req):
            called[0] = True
            return HTTPUnauthorized(request=req)
        with save_globals():
            self.app.object_post_as_copy = False
            set_http_connect(200, 200, 201, 201, 201)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
                                headers={'Content-Length': '5'}, body='12345')
            req.environ['swift.authorize'] = authorize
            self.app.update_request(req)
            res = controller.POST(req)
        self.assert_(called[0])

    def test_POST_as_copy_calls_authorize(self):
        called = [False]

        def authorize(req):
            called[0] = True
            return HTTPUnauthorized(request=req)
        with save_globals():
            set_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
                                headers={'Content-Length': '5'}, body='12345')
            req.environ['swift.authorize'] = authorize
            self.app.update_request(req)
            res = controller.POST(req)
        self.assert_(called[0])

    def test_PUT_calls_authorize(self):
        called = [False]

        def authorize(req):
            called[0] = True
            return HTTPUnauthorized(request=req)
        with save_globals():
            set_http_connect(200, 200, 201, 201, 201)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'Content-Length': '5'}, body='12345')
            req.environ['swift.authorize'] = authorize
            self.app.update_request(req)
            res = controller.PUT(req)
        self.assert_(called[0])

    def test_COPY_calls_authorize(self):
        called = [False]

        def authorize(req):
            called[0] = True
            return HTTPUnauthorized(request=req)
        with save_globals():
            set_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
                                headers={'Destination': 'c/o'})
            req.environ['swift.authorize'] = authorize
            self.app.update_request(req)
            res = controller.COPY(req)
        self.assert_(called[0])

    def test_POST_converts_delete_after_to_delete_at(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 200, 200, 200, 202, 202, 202)
            self.app.memcache.store = {}
            orig_time = time.time
            try:
                t = time.time()
                time.time = lambda: t
                req = Request.blank('/a/c/o', {},
                                    headers={'Content-Type': 'foo/bar',
                                             'X-Delete-After': '60'})
                self.app.update_request(req)
                res = controller.POST(req)
                self.assertEquals(res.status, '202 Fake')
                self.assertEquals(req.headers.get('x-delete-at'),
                                  str(int(t + 60)))

                self.app.object_post_as_copy = False
                controller = proxy_server.ObjectController(self.app, 'account',
                                                           'container',
                                                           'object')
                set_http_connect(200, 200, 202, 202, 202)
                self.app.memcache.store = {}
                req = Request.blank('/a/c/o', {},
                                    headers={'Content-Type': 'foo/bar',
                                             'X-Delete-After': '60'})
                self.app.update_request(req)
                res = controller.POST(req)
                self.assertEquals(res.status, '202 Fake')
                self.assertEquals(req.headers.get('x-delete-at'),
                                  str(int(t + 60)))
            finally:
                time.time = orig_time

    def test_POST_non_int_delete_after(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 200, 200, 200, 202, 202, 202)
            self.app.memcache.store = {}
            req = Request.blank('/a/c/o', {},
                                headers={'Content-Type': 'foo/bar',
                                         'X-Delete-After': '60.1'})
            self.app.update_request(req)
            res = controller.POST(req)
            self.assertEquals(res.status, '400 Bad Request')
            self.assertTrue('Non-integer X-Delete-After' in res.body)

    def test_POST_negative_delete_after(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 200, 200, 200, 202, 202, 202)
            self.app.memcache.store = {}
            req = Request.blank('/a/c/o', {},
                                headers={'Content-Type': 'foo/bar',
                                         'X-Delete-After': '-60'})
            self.app.update_request(req)
            res = controller.POST(req)
            self.assertEquals(res.status, '400 Bad Request')
            self.assertTrue('X-Delete-At in past' in res.body)

    def test_POST_delete_at(self):
        with save_globals():
            given_headers = {}

            def fake_make_requests(req, ring, part, method, path, headers,
                                   query_string=''):
                given_headers.update(headers[0])

            self.app.object_post_as_copy = False
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            controller.make_requests = fake_make_requests
            set_http_connect(200, 200)
            self.app.memcache.store = {}
            t = str(int(time.time() + 100))
            req = Request.blank('/a/c/o', {},
                                headers={'Content-Type': 'foo/bar',
                                         'X-Delete-At': t})
            self.app.update_request(req)
            controller.POST(req)
            self.assertEquals(given_headers.get('X-Delete-At'), t)
            self.assertTrue('X-Delete-At-Host' in given_headers)
            self.assertTrue('X-Delete-At-Device' in given_headers)
            self.assertTrue('X-Delete-At-Partition' in given_headers)

            t = str(int(time.time() + 100)) + '.1'
            req = Request.blank('/a/c/o', {},
                                headers={'Content-Type': 'foo/bar',
                                         'X-Delete-At': t})
            self.app.update_request(req)
            resp = controller.POST(req)
            self.assertEquals(resp.status_int, 400)
            self.assertTrue('Non-integer X-Delete-At' in resp.body)

            t = str(int(time.time() - 100))
            req = Request.blank('/a/c/o', {},
                                headers={'Content-Type': 'foo/bar',
                                         'X-Delete-At': t})
            self.app.update_request(req)
            resp = controller.POST(req)
            self.assertEquals(resp.status_int, 400)
            self.assertTrue('X-Delete-At in past' in resp.body)

    def test_PUT_converts_delete_after_to_delete_at(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 201, 201, 201)
            self.app.memcache.store = {}
            orig_time = time.time
            try:
                t = time.time()
                time.time = lambda: t
                req = Request.blank('/a/c/o', {},
                                    headers={'Content-Length': '0',
                                             'Content-Type': 'foo/bar',
                                             'X-Delete-After': '60'})
                self.app.update_request(req)
                res = controller.PUT(req)
                self.assertEquals(res.status, '201 Fake')
                self.assertEquals(req.headers.get('x-delete-at'),
                                  str(int(t + 60)))
            finally:
                time.time = orig_time

    def test_PUT_non_int_delete_after(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 201, 201, 201)
            self.app.memcache.store = {}
            req = Request.blank('/a/c/o', {},
                                headers={'Content-Length': '0',
                                         'Content-Type': 'foo/bar',
                                         'X-Delete-After': '60.1'})
            self.app.update_request(req)
            res = controller.PUT(req)
            self.assertEquals(res.status, '400 Bad Request')
            self.assertTrue('Non-integer X-Delete-After' in res.body)

    def test_PUT_negative_delete_after(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            set_http_connect(200, 200, 201, 201, 201)
            self.app.memcache.store = {}
            req = Request.blank('/a/c/o', {},
                                headers={'Content-Length': '0',
                                         'Content-Type': 'foo/bar',
                                         'X-Delete-After': '-60'})
            self.app.update_request(req)
            res = controller.PUT(req)
            self.assertEquals(res.status, '400 Bad Request')
            self.assertTrue('X-Delete-At in past' in res.body)

    def test_PUT_delete_at(self):
        with save_globals():
            given_headers = {}

            def fake_connect_put_node(nodes, part, path, headers,
                                      logger_thread_locals):
                given_headers.update(headers)

            controller = proxy_server.ObjectController(self.app, 'account',
                                                       'container', 'object')
            controller._connect_put_node = fake_connect_put_node
            set_http_connect(200, 200)
            self.app.memcache.store = {}
            t = str(int(time.time() + 100))
            req = Request.blank('/a/c/o', {},
                                headers={'Content-Length': '0',
                                         'Content-Type': 'foo/bar',
                                         'X-Delete-At': t})
            self.app.update_request(req)
            controller.PUT(req)
            self.assertEquals(given_headers.get('X-Delete-At'), t)
            self.assertTrue('X-Delete-At-Host' in given_headers)
            self.assertTrue('X-Delete-At-Device' in given_headers)
            self.assertTrue('X-Delete-At-Partition' in given_headers)

            t = str(int(time.time() + 100)) + '.1'
            req = Request.blank('/a/c/o', {},
                                headers={'Content-Length': '0',
                                         'Content-Type': 'foo/bar',
                                         'X-Delete-At': t})
            self.app.update_request(req)
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 400)
            self.assertTrue('Non-integer X-Delete-At' in resp.body)

            t = str(int(time.time() - 100))
            req = Request.blank('/a/c/o', {},
                                headers={'Content-Length': '0',
                                         'Content-Type': 'foo/bar',
                                         'X-Delete-At': t})
            self.app.update_request(req)
            resp = controller.PUT(req)
            self.assertEquals(resp.status_int, 400)
            self.assertTrue('X-Delete-At in past' in resp.body)

    def test_leak_1(self):
        global _request_instances
        prolis = _test_sockets[0]
        prosrv = _test_servers[0]
        obj_len = prosrv.client_chunk_size * 2
        # PUT test file
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('PUT /v1/a/c/test_leak_1 HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Auth-Token: t\r\n'
                 'Content-Length: %s\r\n'
                 'Content-Type: application/octet-stream\r\n'
                 '\r\n%s' % (obj_len, 'a' * obj_len))
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 201'
        self.assertEqual(headers[:len(exp)], exp)
        # Remember Request instance count
        before_request_instances = _request_instances
        # GET test file, but disconnect early
        sock = connect_tcp(('localhost', prolis.getsockname()[1]))
        fd = sock.makefile()
        fd.write('GET /v1/a/c/test_leak_1 HTTP/1.1\r\n'
                 'Host: localhost\r\n'
                 'Connection: close\r\n'
                 'X-Auth-Token: t\r\n'
                 '\r\n')
        fd.flush()
        headers = readuntil2crlfs(fd)
        exp = 'HTTP/1.1 200'
        self.assertEqual(headers[:len(exp)], exp)
        fd.read(1)
        fd.close()
        sock.close()
        self.assertEquals(before_request_instances, _request_instances)

    def test_OPTIONS(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'a',
                                                       'c', 'o.jpg')

            def my_empty_container_info(*args):
                return {}
            controller.container_info = my_empty_container_info
            req = Request.blank(
                '/a/c/o.jpg',
                {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'http://foo.com',
                         'Access-Control-Request-Method': 'GET'})
            resp = controller.OPTIONS(req)
            self.assertEquals(401, resp.status_int)

            def my_empty_origin_container_info(*args):
                return {'cors': {'allow_origin': None}}
            controller.container_info = my_empty_origin_container_info
            req = Request.blank(
                '/a/c/o.jpg',
                {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'http://foo.com',
                         'Access-Control-Request-Method': 'GET'})
            resp = controller.OPTIONS(req)
            self.assertEquals(401, resp.status_int)

            def my_container_info(*args):
                return {
                    'cors': {
                        'allow_origin': 'http://foo.bar:8080 https://foo.bar',
                        'max_age': '999',
                    }
                }
            controller.container_info = my_container_info
            req = Request.blank(
                '/a/c/o.jpg',
                {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'https://foo.bar',
                         'Access-Control-Request-Method': 'GET'})
            req.content_length = 0
            resp = controller.OPTIONS(req)
            self.assertEquals(200, resp.status_int)
            self.assertEquals(
                'https://foo.bar',
                resp.headers['access-control-allow-origin'])
            for verb in 'OPTIONS COPY GET POST PUT DELETE HEAD'.split():
                self.assertTrue(
                    verb in resp.headers['access-control-allow-methods'])
            self.assertEquals(
                len(resp.headers['access-control-allow-methods'].split(', ')),
                7)
            self.assertEquals('999', resp.headers['access-control-max-age'])
            req = Request.blank(
                '/a/c/o.jpg',
                {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'https://foo.bar'})
            req.content_length = 0
            resp = controller.OPTIONS(req)
            self.assertEquals(401, resp.status_int)
            req = Request.blank('/a/c/o.jpg', {'REQUEST_METHOD': 'OPTIONS'})
            req.content_length = 0
            resp = controller.OPTIONS(req)
            self.assertEquals(200, resp.status_int)
            for verb in 'OPTIONS COPY GET POST PUT DELETE HEAD'.split():
                self.assertTrue(
                    verb in resp.headers['Allow'])
            self.assertEquals(len(resp.headers['Allow'].split(', ')), 7)
            req = Request.blank(
                '/a/c/o.jpg',
                {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'http://foo.com'})
            resp = controller.OPTIONS(req)
            self.assertEquals(401, resp.status_int)
            req = Request.blank(
                '/a/c/o.jpg',
                {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'http://foo.bar',
                         'Access-Control-Request-Method': 'GET'})
            controller.app.cors_allow_origin = ['http://foo.bar', ]
            resp = controller.OPTIONS(req)
            self.assertEquals(200, resp.status_int)

            def my_container_info_wildcard(*args):
                return {
                    'cors': {
                        'allow_origin': '*',
                        'max_age': '999',
                    }
                }
            controller.container_info = my_container_info_wildcard
            req = Request.blank(
                '/a/c/o.jpg',
                {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'https://bar.baz',
                         'Access-Control-Request-Method': 'GET'})
            req.content_length = 0
            resp = controller.OPTIONS(req)
            self.assertEquals(200, resp.status_int)
            self.assertEquals(
                'https://bar.baz',
                resp.headers['access-control-allow-origin'])
            for verb in 'OPTIONS COPY GET POST PUT DELETE HEAD'.split():
                self.assertTrue(
                    verb in resp.headers['access-control-allow-methods'])
            self.assertEquals(
                len(resp.headers['access-control-allow-methods'].split(', ')),
                7)
            self.assertEquals('999', resp.headers['access-control-max-age'])

    def test_CORS_valid(self):
        with save_globals():
            controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')

            def stubContainerInfo(*args):
                return {
                    'cors': {
                        'allow_origin': 'http://foo.bar'
                    }
                }
            controller.container_info = stubContainerInfo

            def objectGET(controller, req):
                return Response(headers={
                    'X-Object-Meta-Color': 'red',
                    'X-Super-Secret': 'hush',
                })

            req = Request.blank(
                '/a/c/o.jpg',
                {'REQUEST_METHOD': 'GET'},
                headers={'Origin': 'http://foo.bar'})

            resp = cors_validation(objectGET)(controller, req)

            self.assertEquals(200, resp.status_int)
            self.assertEquals('http://foo.bar',
                              resp.headers['access-control-allow-origin'])
            self.assertEquals('red', resp.headers['x-object-meta-color'])
            # X-Super-Secret is in the response, but not "exposed"
            self.assertEquals('hush', resp.headers['x-super-secret'])
            self.assertTrue('access-control-expose-headers' in resp.headers)
            exposed = set(
                h.strip() for h in
                resp.headers['access-control-expose-headers'].split(','))
            expected_exposed = set(['cache-control', 'content-language',
                                    'content-type', 'expires', 'last-modified',
                                    'pragma', 'etag', 'x-timestamp',
                                    'x-trans-id', 'x-object-meta-color'])
            self.assertEquals(expected_exposed, exposed)

    def _gather_x_container_headers(self, controller_call, req, *connect_args,
                                    **kwargs):
        header_list = kwargs.pop('header_list', ['X-Container-Partition',
                                                 'X-Container-Host',
                                                 'X-Container-Device'])
        seen_headers = []

        def capture_headers(ipaddr, port, device, partition, method,
                            path, headers=None, query_string=None):
            captured = {}
            for header in header_list:
                captured[header] = headers.get(header)
            seen_headers.append(captured)

        with save_globals():
            self.app.allow_account_management = True

            set_http_connect(*connect_args, give_connect=capture_headers,
                             **kwargs)
            resp = controller_call(req)
            self.assertEqual(2, resp.status_int // 100)  # sanity check

            # don't care about the account/container HEADs, so chuck
            # the first two requests
            return sorted(seen_headers[2:],
                          key=lambda d: d.get(header_list[0]) or 'Z')

    def test_PUT_x_container_headers_with_equal_replicas(self):
        req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                            headers={'Content-Length': '5'}, body='12345')
        controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
        seen_headers = self._gather_x_container_headers(
            controller.PUT, req,
            200, 200, 201, 201, 201)   # HEAD HEAD PUT PUT PUT
        self.assertEqual(seen_headers, [
                {'X-Container-Host': '10.0.0.0:1000',
                 'X-Container-Partition': 1,
                 'X-Container-Device': 'sda'},
                {'X-Container-Host': '10.0.0.1:1001',
                 'X-Container-Partition': 1,
                 'X-Container-Device': 'sdb'},
                {'X-Container-Host': '10.0.0.2:1002',
                 'X-Container-Partition': 1,
                 'X-Container-Device': 'sdc'}])

    def test_PUT_x_container_headers_with_fewer_container_replicas(self):
        self.app.container_ring.set_replicas(2)

        req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                            headers={'Content-Length': '5'}, body='12345')
        controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
        seen_headers = self._gather_x_container_headers(
            controller.PUT, req,
            200, 200, 201, 201, 201)   # HEAD HEAD PUT PUT PUT

        self.assertEqual(seen_headers, [
                {'X-Container-Host': '10.0.0.0:1000',
                 'X-Container-Partition': 1,
                 'X-Container-Device': 'sda'},
                {'X-Container-Host': '10.0.0.1:1001',
                 'X-Container-Partition': 1,
                 'X-Container-Device': 'sdb'},
                {'X-Container-Host': None,
                 'X-Container-Partition': None,
                 'X-Container-Device': None}])

    def test_PUT_x_container_headers_with_more_container_replicas(self):
        self.app.container_ring.set_replicas(4)

        req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                            headers={'Content-Length': '5'}, body='12345')
        controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
        seen_headers = self._gather_x_container_headers(
            controller.PUT, req,
            200, 200, 201, 201, 201)   # HEAD HEAD PUT PUT PUT

        self.assertEqual(seen_headers, [
                {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003',
                 'X-Container-Partition': 1,
                 'X-Container-Device': 'sda,sdd'},
                {'X-Container-Host': '10.0.0.1:1001',
                 'X-Container-Partition': 1,
                 'X-Container-Device': 'sdb'},
                {'X-Container-Host': '10.0.0.2:1002',
                 'X-Container-Partition': 1,
                 'X-Container-Device': 'sdc'}])

    def test_POST_x_container_headers_with_more_container_replicas(self):
        self.app.container_ring.set_replicas(4)
        self.app.object_post_as_copy = False

        req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
                            headers={'Content-Type': 'application/stuff'})
        controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
        seen_headers = self._gather_x_container_headers(
            controller.POST, req,
            200, 200, 200, 200, 200)   # HEAD HEAD POST POST POST

        self.assertEqual(seen_headers, [
                {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003',
                 'X-Container-Partition': 1,
                 'X-Container-Device': 'sda,sdd'},
                {'X-Container-Host': '10.0.0.1:1001',
                 'X-Container-Partition': 1,
                 'X-Container-Device': 'sdb'},
                {'X-Container-Host': '10.0.0.2:1002',
                 'X-Container-Partition': 1,
                 'X-Container-Device': 'sdc'}])

    def test_DELETE_x_container_headers_with_more_container_replicas(self):
        self.app.container_ring.set_replicas(4)

        req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'},
                            headers={'Content-Type': 'application/stuff'})
        controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
        seen_headers = self._gather_x_container_headers(
            controller.DELETE, req,
            200, 200, 200, 200, 200)   # HEAD HEAD DELETE DELETE DELETE

        self.assertEqual(seen_headers, [
                {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003',
                 'X-Container-Partition': 1,
                 'X-Container-Device': 'sda,sdd'},
                {'X-Container-Host': '10.0.0.1:1001',
                 'X-Container-Partition': 1,
                 'X-Container-Device': 'sdb'},
                {'X-Container-Host': '10.0.0.2:1002',
                 'X-Container-Partition': 1,
                 'X-Container-Device': 'sdc'}])

    def test_PUT_x_delete_at_with_fewer_container_replicas(self):
        self.app.container_ring.set_replicas(2)

        req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                            headers={'Content-Type': 'application/stuff',
                                     'Content-Length': '0',
                                     'X-Delete-At': int(time.time()) + 100000})
        controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
        seen_headers = self._gather_x_container_headers(
            controller.PUT, req,
            200, 200, 201, 201, 201,   # HEAD HEAD PUT PUT PUT
            header_list=('X-Delete-At-Host', 'X-Delete-At-Device',
                         'X-Delete-At-Partition'))

        self.assertEqual(seen_headers, [
                {'X-Delete-At-Host': '10.0.0.0:1000',
                 'X-Delete-At-Partition': 1,
                 'X-Delete-At-Device': 'sda'},
                {'X-Delete-At-Host': '10.0.0.1:1001',
                 'X-Delete-At-Partition': 1,
                 'X-Delete-At-Device': 'sdb'},
                {'X-Delete-At-Host': None,
                 'X-Delete-At-Partition': None,
                 'X-Delete-At-Device': None}])

    def test_PUT_x_delete_at_with_more_container_replicas(self):
        self.app.container_ring.set_replicas(4)
        self.app.expiring_objects_account = 'expires'
        self.app.expiring_objects_container_divisor = 60

        req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                            headers={'Content-Type': 'application/stuff',
                                     'Content-Length': 0,
                                     'X-Delete-At': int(time.time()) + 100000})
        controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
        seen_headers = self._gather_x_container_headers(
            controller.PUT, req,
            200, 200, 201, 201, 201,   # HEAD HEAD PUT PUT PUT
            header_list=('X-Delete-At-Host', 'X-Delete-At-Device',
                         'X-Delete-At-Partition'))
        self.assertEqual(seen_headers, [
                {'X-Delete-At-Host': '10.0.0.0:1000,10.0.0.3:1003',
                 'X-Delete-At-Partition': 1,
                 'X-Delete-At-Device': 'sda,sdd'},
                {'X-Delete-At-Host': '10.0.0.1:1001',
                 'X-Delete-At-Partition': 1,
                 'X-Delete-At-Device': 'sdb'},
                {'X-Delete-At-Host': '10.0.0.2:1002',
                 'X-Delete-At-Partition': 1,
                 'X-Delete-At-Device': 'sdc'}])


class TestContainerController(unittest.TestCase):
    "Test swift.proxy_server.ContainerController"

    def setUp(self):
        self.app = proxy_server.Application(None, FakeMemcache(),
                                            account_ring=FakeRing(),
                                            container_ring=FakeRing(),
                                            object_ring=FakeRing())

    def test_transfer_headers(self):
        src_headers = {'x-remove-versions-location': 'x',
                       'x-container-read': '*:user'}
        dst_headers = {'x-versions-location': 'backup'}
        controller = swift.proxy.controllers.ContainerController(self.app,
                                                                 'a', 'c')
        controller.transfer_headers(src_headers, dst_headers)
        expected_headers = {'x-versions-location': '',
                            'x-container-read': '*:user'}
        self.assertEqual(dst_headers, expected_headers)

    def assert_status_map(self, method, statuses, expected,
                          raise_exc=False, missing_container=False):
        with save_globals():
            kwargs = {}
            if raise_exc:
                kwargs['raise_exc'] = raise_exc
            kwargs['missing_container'] = missing_container
            set_http_connect(*statuses, **kwargs)
            self.app.memcache.store = {}
            req = Request.blank('/a/c', headers={'Content-Length': '0',
                                'Content-Type': 'text/plain'})
            self.app.update_request(req)
            res = method(req)
            self.assertEquals(res.status_int, expected)
            set_http_connect(*statuses, **kwargs)
            self.app.memcache.store = {}
            req = Request.blank('/a/c/', headers={'Content-Length': '0',
                                'Content-Type': 'text/plain'})
            self.app.update_request(req)
            res = method(req)
            self.assertEquals(res.status_int, expected)

    def test_HEAD(self):
        with save_globals():
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')

            def test_status_map(statuses, expected, **kwargs):
                set_http_connect(*statuses, **kwargs)
                self.app.memcache.store = {}
                req = Request.blank('/a/c', {})
                self.app.update_request(req)
                res = controller.HEAD(req)
                self.assertEquals(res.status[:len(str(expected))],
                                  str(expected))
                if expected < 400:
                    self.assert_('x-works' in res.headers)
                    self.assertEquals(res.headers['x-works'], 'yes')
            test_status_map((200, 200, 404, 404), 200)
            test_status_map((200, 200, 500, 404), 200)
            test_status_map((200, 304, 500, 404), 304)
            test_status_map((200, 404, 404, 404), 404)
            test_status_map((200, 404, 404, 500), 404)
            test_status_map((200, 500, 500, 500), 503)

    def test_PUT(self):
        with save_globals():
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')

            def test_status_map(statuses, expected, **kwargs):
                set_http_connect(*statuses, **kwargs)
                self.app.memcache.store = {}
                req = Request.blank('/a/c', {})
                req.content_length = 0
                self.app.update_request(req)
                res = controller.PUT(req)
                expected = str(expected)
                self.assertEquals(res.status[:len(expected)], expected)
            test_status_map((200, 201, 201, 201), 201, missing_container=True)
            test_status_map((200, 201, 201, 500), 201, missing_container=True)
            test_status_map((200, 204, 404, 404), 404, missing_container=True)
            test_status_map((200, 204, 500, 404), 503, missing_container=True)

    def test_PUT_max_containers_per_account(self):
        with save_globals():
            self.app.max_containers_per_account = 12346
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            self.assert_status_map(controller.PUT,
                                   (200, 200, 200, 201, 201, 201), 201,
                                   missing_container=True)

            self.app.max_containers_per_account = 12345
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            self.assert_status_map(controller.PUT, (201, 201, 201), 403,
                                   missing_container=True)

            self.app.max_containers_per_account = 12345
            self.app.max_containers_whitelist = ['account']
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            self.assert_status_map(controller.PUT,
                                   (200, 200, 200, 201, 201, 201), 201,
                                   missing_container=True)

    def test_PUT_max_container_name_length(self):
        with save_globals():
            limit = MAX_CONTAINER_NAME_LENGTH
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          '1' * limit)
            self.assert_status_map(controller.PUT,
                                   (200, 200, 200, 201, 201, 201), 201,
                                   missing_container=True)
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          '2' * (limit + 1))
            self.assert_status_map(controller.PUT, (201, 201, 201), 400,
                                   missing_container=True)

    def test_PUT_connect_exceptions(self):
        with save_globals():
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            self.assert_status_map(controller.PUT, (200, 201, 201, -1), 201,
                                   missing_container=True)
            self.assert_status_map(controller.PUT, (200, 201, -1, -1), 503,
                                   missing_container=True)
            self.assert_status_map(controller.PUT, (200, 503, 503, -1), 503,
                                   missing_container=True)

    def test_acc_missing_returns_404(self):
        for meth in ('DELETE', 'PUT'):
            with save_globals():
                self.app.memcache = FakeMemcacheReturnsNone()
                for dev in self.app.account_ring.devs.values():
                    del dev['errors']
                    del dev['last_error']
                controller = proxy_server.ContainerController(self.app,
                                                              'account',
                                                              'container')
                if meth == 'PUT':
                    set_http_connect(200, 200, 200, 200, 200, 200,
                                     missing_container=True)
                else:
                    set_http_connect(200, 200, 200, 200)
                self.app.memcache.store = {}
                req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth})
                self.app.update_request(req)
                resp = getattr(controller, meth)(req)
                self.assertEquals(resp.status_int, 200)

                set_http_connect(404, 404, 404, 200, 200, 200)
                resp = getattr(controller, meth)(req)
                self.assertEquals(resp.status_int, 404)

                set_http_connect(503, 404, 404)
                resp = getattr(controller, meth)(req)
                self.assertEquals(resp.status_int, 404)

                set_http_connect(503, 404, raise_exc=True)
                resp = getattr(controller, meth)(req)
                self.assertEquals(resp.status_int, 404)

                for dev in self.app.account_ring.devs.values():
                    dev['errors'] = self.app.error_suppression_limit + 1
                    dev['last_error'] = time.time()
                set_http_connect(200, 200, 200, 200, 200, 200)
                resp = getattr(controller, meth)(req)
                self.assertEquals(resp.status_int, 404)

    def test_put_locking(self):

        class MockMemcache(FakeMemcache):

            def __init__(self, allow_lock=None):
                self.allow_lock = allow_lock
                super(MockMemcache, self).__init__()

            @contextmanager
            def soft_lock(self, key, timeout=0, retries=5):
                if self.allow_lock:
                    yield True
                else:
                    raise MemcacheLockError()

        with save_globals():
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            self.app.memcache = MockMemcache(allow_lock=True)
            set_http_connect(200, 200, 200, 201, 201, 201,
                             missing_container=True)
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'})
            self.app.update_request(req)
            res = controller.PUT(req)
            self.assertEquals(res.status_int, 201)

    def test_error_limiting(self):
        with save_globals():
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            controller.app.sort_nodes = lambda l: l
            self.assert_status_map(controller.HEAD, (200, 503, 200, 200), 200,
                                   missing_container=False)
            self.assertEquals(
                controller.app.container_ring.devs[0]['errors'], 2)
            self.assert_('last_error' in controller.app.container_ring.devs[0])
            for _junk in xrange(self.app.error_suppression_limit):
                self.assert_status_map(controller.HEAD,
                                       (200, 503, 503, 503), 503)
            self.assertEquals(controller.app.container_ring.devs[0]['errors'],
                              self.app.error_suppression_limit + 1)
            self.assert_status_map(controller.HEAD, (200, 200, 200, 200), 503)
            self.assert_('last_error' in controller.app.container_ring.devs[0])
            self.assert_status_map(controller.PUT, (200, 201, 201, 201), 503,
                                   missing_container=True)
            self.assert_status_map(controller.DELETE,
                                   (200, 204, 204, 204), 503)
            self.app.error_suppression_interval = -300
            self.assert_status_map(controller.HEAD, (200, 200, 200, 200), 200)
            self.assert_status_map(controller.DELETE, (200, 204, 204, 204),
                                   404, raise_exc=True)

    def test_DELETE(self):
        with save_globals():
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            self.assert_status_map(controller.DELETE,
                                   (200, 204, 204, 204), 204)
            self.assert_status_map(controller.DELETE,
                                   (200, 204, 204, 503), 204)
            self.assert_status_map(controller.DELETE,
                                   (200, 204, 503, 503), 503)
            self.assert_status_map(controller.DELETE,
                                   (200, 204, 404, 404), 404)
            self.assert_status_map(controller.DELETE,
                                   (200, 404, 404, 404), 404)
            self.assert_status_map(controller.DELETE,
                                   (200, 204, 503, 404), 503)

            self.app.memcache = FakeMemcacheReturnsNone()
            # 200: Account check, 404x3: Container check
            self.assert_status_map(controller.DELETE,
                                   (200, 404, 404, 404), 404)

    def test_response_get_accept_ranges_header(self):
        with save_globals():
            set_http_connect(200, 200, body='{}')
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            req = Request.blank('/a/c?format=json')
            self.app.update_request(req)
            res = controller.GET(req)
            self.assert_('accept-ranges' in res.headers)
            self.assertEqual(res.headers['accept-ranges'], 'bytes')

    def test_response_head_accept_ranges_header(self):
        with save_globals():
            set_http_connect(200, 200, body='{}')
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            req = Request.blank('/a/c?format=json')
            self.app.update_request(req)
            res = controller.HEAD(req)
            self.assert_('accept-ranges' in res.headers)
            self.assertEqual(res.headers['accept-ranges'], 'bytes')

    def test_PUT_metadata(self):
        self.metadata_helper('PUT')

    def test_POST_metadata(self):
        self.metadata_helper('POST')

    def metadata_helper(self, method):
        for test_header, test_value in (
                ('X-Container-Meta-TestHeader', 'TestValue'),
                ('X-Container-Meta-TestHeader', ''),
                ('X-Remove-Container-Meta-TestHeader', 'anything'),
                ('X-Container-Read', '.r:*'),
                ('X-Remove-Container-Read', 'anything'),
                ('X-Container-Write', 'anyone'),
                ('X-Remove-Container-Write', 'anything')):
            test_errors = []

            def test_connect(ipaddr, port, device, partition, method, path,
                             headers=None, query_string=None):
                if path == '/a/c':
                    find_header = test_header
                    find_value = test_value
                    if find_header.lower().startswith('x-remove-'):
                        find_header = \
                            find_header.lower().replace('-remove', '', 1)
                        find_value = ''
                    for k, v in headers.iteritems():
                        if k.lower() == find_header.lower() and \
                                v == find_value:
                            break
                    else:
                        test_errors.append('%s: %s not in %s' %
                                           (find_header, find_value, headers))
            with save_globals():
                controller = \
                    proxy_server.ContainerController(self.app, 'a', 'c')
                set_http_connect(200, 201, 201, 201, give_connect=test_connect)
                req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                    headers={test_header: test_value})
                self.app.update_request(req)
                res = getattr(controller, method)(req)
                self.assertEquals(test_errors, [])

    def test_PUT_bad_metadata(self):
        self.bad_metadata_helper('PUT')

    def test_POST_bad_metadata(self):
        self.bad_metadata_helper('POST')

    def bad_metadata_helper(self, method):
        with save_globals():
            controller = proxy_server.ContainerController(self.app, 'a', 'c')
            set_http_connect(200, 201, 201, 201)
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method})
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 201)

            set_http_connect(201, 201, 201)
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers={'X-Container-Meta-' +
                                ('a' * MAX_META_NAME_LENGTH): 'v'})
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 201)
            set_http_connect(201, 201, 201)
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers={'X-Container-Meta-' +
                                ('a' * (MAX_META_NAME_LENGTH + 1)): 'v'})
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 400)

            set_http_connect(201, 201, 201)
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers={'X-Container-Meta-Too-Long':
                                'a' * MAX_META_VALUE_LENGTH})
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 201)
            set_http_connect(201, 201, 201)
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers={'X-Container-Meta-Too-Long':
                                'a' * (MAX_META_VALUE_LENGTH + 1)})
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 400)

            set_http_connect(201, 201, 201)
            headers = {}
            for x in xrange(MAX_META_COUNT):
                headers['X-Container-Meta-%d' % x] = 'v'
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers=headers)
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 201)
            set_http_connect(201, 201, 201)
            headers = {}
            for x in xrange(MAX_META_COUNT + 1):
                headers['X-Container-Meta-%d' % x] = 'v'
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers=headers)
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 400)

            set_http_connect(201, 201, 201)
            headers = {}
            header_value = 'a' * MAX_META_VALUE_LENGTH
            size = 0
            x = 0
            while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH:
                size += 4 + MAX_META_VALUE_LENGTH
                headers['X-Container-Meta-%04d' % x] = header_value
                x += 1
            if MAX_META_OVERALL_SIZE - size > 1:
                headers['X-Container-Meta-a'] = \
                    'a' * (MAX_META_OVERALL_SIZE - size - 1)
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers=headers)
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 201)
            set_http_connect(201, 201, 201)
            headers['X-Container-Meta-a'] = \
                'a' * (MAX_META_OVERALL_SIZE - size)
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers=headers)
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 400)

    def test_POST_calls_clean_acl(self):
        called = [False]

        def clean_acl(header, value):
            called[0] = True
            raise ValueError('fake error')
        with save_globals():
            set_http_connect(200, 201, 201, 201)
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'POST'},
                                headers={'X-Container-Read': '.r:*'})
            req.environ['swift.clean_acl'] = clean_acl
            self.app.update_request(req)
            res = controller.POST(req)
        self.assert_(called[0])
        called[0] = False
        with save_globals():
            set_http_connect(200, 201, 201, 201)
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'POST'},
                                headers={'X-Container-Write': '.r:*'})
            req.environ['swift.clean_acl'] = clean_acl
            self.app.update_request(req)
            res = controller.POST(req)
        self.assert_(called[0])

    def test_PUT_calls_clean_acl(self):
        called = [False]

        def clean_acl(header, value):
            called[0] = True
            raise ValueError('fake error')
        with save_globals():
            set_http_connect(200, 201, 201, 201)
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'X-Container-Read': '.r:*'})
            req.environ['swift.clean_acl'] = clean_acl
            self.app.update_request(req)
            res = controller.PUT(req)
        self.assert_(called[0])
        called[0] = False
        with save_globals():
            set_http_connect(200, 201, 201, 201)
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'},
                                headers={'X-Container-Write': '.r:*'})
            req.environ['swift.clean_acl'] = clean_acl
            self.app.update_request(req)
            res = controller.PUT(req)
        self.assert_(called[0])

    def test_GET_no_content(self):
        with save_globals():
            set_http_connect(200, 204, 204, 204)
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            req = Request.blank('/a/c')
            self.app.update_request(req)
            res = controller.GET(req)
            self.assertEquals(res.content_length, 0)
            self.assertTrue('transfer-encoding' not in res.headers)

    def test_GET_calls_authorize(self):
        called = [False]

        def authorize(req):
            called[0] = True
            return HTTPUnauthorized(request=req)
        with save_globals():
            set_http_connect(200, 201, 201, 201)
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            req = Request.blank('/a/c')
            req.environ['swift.authorize'] = authorize
            self.app.update_request(req)
            res = controller.GET(req)
        self.assert_(called[0])

    def test_HEAD_calls_authorize(self):
        called = [False]

        def authorize(req):
            called[0] = True
            return HTTPUnauthorized(request=req)
        with save_globals():
            set_http_connect(200, 201, 201, 201)
            controller = proxy_server.ContainerController(self.app, 'account',
                                                          'container')
            req = Request.blank('/a/c', {'REQUEST_METHOD': 'HEAD'})
            req.environ['swift.authorize'] = authorize
            self.app.update_request(req)
            res = controller.HEAD(req)
        self.assert_(called[0])

    def test_OPTIONS(self):
        with save_globals():
            controller = proxy_server.ContainerController(self.app, 'a', 'c')

            def my_empty_container_info(*args):
                return {}
            controller.container_info = my_empty_container_info
            req = Request.blank(
                '/a/c',
                {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'http://foo.com',
                         'Access-Control-Request-Method': 'GET'})
            resp = controller.OPTIONS(req)
            self.assertEquals(401, resp.status_int)

            def my_empty_origin_container_info(*args):
                return {'cors': {'allow_origin': None}}
            controller.container_info = my_empty_origin_container_info
            req = Request.blank(
                '/a/c',
                {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'http://foo.com',
                         'Access-Control-Request-Method': 'GET'})
            resp = controller.OPTIONS(req)
            self.assertEquals(401, resp.status_int)

            def my_container_info(*args):
                return {
                    'cors': {
                        'allow_origin': 'http://foo.bar:8080 https://foo.bar',
                        'max_age': '999',
                    }
                }
            controller.container_info = my_container_info
            req = Request.blank(
                '/a/c',
                {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'https://foo.bar',
                         'Access-Control-Request-Method': 'GET'})
            req.content_length = 0
            resp = controller.OPTIONS(req)
            self.assertEquals(200, resp.status_int)
            self.assertEquals(
                'https://foo.bar',
                resp.headers['access-control-allow-origin'])
            for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split():
                self.assertTrue(
                    verb in resp.headers['access-control-allow-methods'])
            self.assertEquals(
                len(resp.headers['access-control-allow-methods'].split(', ')),
                6)
            self.assertEquals('999', resp.headers['access-control-max-age'])
            req = Request.blank(
                '/a/c',
                {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'https://foo.bar'})
            req.content_length = 0
            resp = controller.OPTIONS(req)
            self.assertEquals(401, resp.status_int)
            req = Request.blank('/a/c', {'REQUEST_METHOD': 'OPTIONS'})
            req.content_length = 0
            resp = controller.OPTIONS(req)
            self.assertEquals(200, resp.status_int)
            for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split():
                self.assertTrue(
                    verb in resp.headers['Allow'])
            self.assertEquals(len(resp.headers['Allow'].split(', ')), 6)
            req = Request.blank(
                '/a/c',
                {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'http://foo.bar',
                         'Access-Control-Request-Method': 'GET'})
            resp = controller.OPTIONS(req)
            self.assertEquals(401, resp.status_int)
            req = Request.blank(
                '/a/c',
                {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'http://foo.bar',
                         'Access-Control-Request-Method': 'GET'})
            controller.app.cors_allow_origin = ['http://foo.bar', ]
            resp = controller.OPTIONS(req)
            self.assertEquals(200, resp.status_int)

            def my_container_info_wildcard(*args):
                return {
                    'cors': {
                        'allow_origin': '*',
                        'max_age': '999',
                    }
                }
            controller.container_info = my_container_info_wildcard
            req = Request.blank(
                '/a/c/o.jpg',
                {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'https://bar.baz',
                         'Access-Control-Request-Method': 'GET'})
            req.content_length = 0
            resp = controller.OPTIONS(req)
            self.assertEquals(200, resp.status_int)
            self.assertEquals(
                'https://bar.baz',
                resp.headers['access-control-allow-origin'])
            for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split():
                self.assertTrue(
                    verb in resp.headers['access-control-allow-methods'])
            self.assertEquals(
                len(resp.headers['access-control-allow-methods'].split(', ')),
                6)
            self.assertEquals('999', resp.headers['access-control-max-age'])

            req = Request.blank(
                '/a/c/o.jpg',
                {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'https://bar.baz',
                         'Access-Control-Request-Headers':
                         'x-foo, x-bar, x-auth-token',
                         'Access-Control-Request-Method': 'GET'}
            )
            req.content_length = 0
            resp = controller.OPTIONS(req)
            self.assertEquals(200, resp.status_int)
            self.assertEquals(
                sortHeaderNames('x-foo, x-bar, x-auth-token'),
                sortHeaderNames(resp.headers['access-control-allow-headers']))

    def test_CORS_valid(self):
        with save_globals():
            controller = proxy_server.ContainerController(self.app, 'a', 'c')

            def stubContainerInfo(*args):
                return {
                    'cors': {
                        'allow_origin': 'http://foo.bar'
                    }
                }
            controller.container_info = stubContainerInfo

            def containerGET(controller, req):
                return Response(headers={
                    'X-Container-Meta-Color': 'red',
                    'X-Super-Secret': 'hush',
                })

            req = Request.blank(
                '/a/c',
                {'REQUEST_METHOD': 'GET'},
                headers={'Origin': 'http://foo.bar'})

            resp = cors_validation(containerGET)(controller, req)

            self.assertEquals(200, resp.status_int)
            self.assertEquals('http://foo.bar',
                              resp.headers['access-control-allow-origin'])
            self.assertEquals('red', resp.headers['x-container-meta-color'])
            # X-Super-Secret is in the response, but not "exposed"
            self.assertEquals('hush', resp.headers['x-super-secret'])
            self.assertTrue('access-control-expose-headers' in resp.headers)
            exposed = set(
                h.strip() for h in
                resp.headers['access-control-expose-headers'].split(','))
            expected_exposed = set(['cache-control', 'content-language',
                                    'content-type', 'expires', 'last-modified',
                                    'pragma', 'etag', 'x-timestamp',
                                    'x-trans-id', 'x-container-meta-color'])
            self.assertEquals(expected_exposed, exposed)

    def _gather_x_account_headers(self, controller_call, req, *connect_args,
                                    **kwargs):
        seen_headers = []
        to_capture = ('X-Account-Partition', 'X-Account-Host',
                      'X-Account-Device')

        def capture_headers(ipaddr, port, device, partition, method,
                            path, headers=None, query_string=None):
            captured = {}
            for header in to_capture:
                captured[header] = headers.get(header)
            seen_headers.append(captured)

        with save_globals():
            self.app.allow_account_management = True

            set_http_connect(*connect_args, give_connect=capture_headers,
                             **kwargs)
            resp = controller_call(req)
            self.assertEqual(2, resp.status_int // 100)  # sanity check

            # don't care about the account HEAD, so throw away the
            # first element
            return sorted(seen_headers[1:],
                          key=lambda d: d['X-Account-Host'] or 'Z')

    def test_PUT_x_account_headers_with_fewer_account_replicas(self):
        self.app.account_ring.set_replicas(2)
        req = Request.blank('/a/c', headers={'': ''})
        controller = proxy_server.ContainerController(self.app, 'a', 'c')

        seen_headers = self._gather_x_account_headers(
            controller.PUT, req,
            200, 201, 201, 201)    # HEAD PUT PUT PUT
        self.assertEqual(seen_headers, [
                {'X-Account-Host': '10.0.0.0:1000',
                 'X-Account-Partition': 1,
                 'X-Account-Device': 'sda'},
                {'X-Account-Host': '10.0.0.1:1001',
                 'X-Account-Partition': 1,
                 'X-Account-Device': 'sdb'},
                {'X-Account-Host': None,
                 'X-Account-Partition': None,
                 'X-Account-Device': None}])

    def test_PUT_x_account_headers_with_more_account_replicas(self):
        self.app.account_ring.set_replicas(4)
        req = Request.blank('/a/c', headers={'': ''})
        controller = proxy_server.ContainerController(self.app, 'a', 'c')

        seen_headers = self._gather_x_account_headers(
            controller.PUT, req,
            200, 201, 201, 201)    # HEAD PUT PUT PUT
        self.assertEqual(seen_headers, [
                {'X-Account-Host': '10.0.0.0:1000,10.0.0.3:1003',
                 'X-Account-Partition': 1,
                 'X-Account-Device': 'sda,sdd'},
                {'X-Account-Host': '10.0.0.1:1001',
                 'X-Account-Partition': 1,
                 'X-Account-Device': 'sdb'},
                {'X-Account-Host': '10.0.0.2:1002',
                 'X-Account-Partition': 1,
                 'X-Account-Device': 'sdc'}])

    def test_DELETE_x_account_headers_with_fewer_account_replicas(self):
        self.app.account_ring.set_replicas(2)
        req = Request.blank('/a/c', headers={'': ''})
        controller = proxy_server.ContainerController(self.app, 'a', 'c')

        seen_headers = self._gather_x_account_headers(
            controller.DELETE, req,
            200, 204, 204, 204)    # HEAD DELETE DELETE DELETE
        self.assertEqual(seen_headers, [
                {'X-Account-Host': '10.0.0.0:1000',
                 'X-Account-Partition': 1,
                 'X-Account-Device': 'sda'},
                {'X-Account-Host': '10.0.0.1:1001',
                 'X-Account-Partition': 1,
                 'X-Account-Device': 'sdb'},
                {'X-Account-Host': None,
                 'X-Account-Partition': None,
                 'X-Account-Device': None}])

    def test_DELETE_x_account_headers_with_more_account_replicas(self):
        self.app.account_ring.set_replicas(4)
        req = Request.blank('/a/c', headers={'': ''})
        controller = proxy_server.ContainerController(self.app, 'a', 'c')

        seen_headers = self._gather_x_account_headers(
            controller.DELETE, req,
            200, 204, 204, 204)    # HEAD DELETE DELETE DELETE
        self.assertEqual(seen_headers, [
                {'X-Account-Host': '10.0.0.0:1000,10.0.0.3:1003',
                 'X-Account-Partition': 1,
                 'X-Account-Device': 'sda,sdd'},
                {'X-Account-Host': '10.0.0.1:1001',
                 'X-Account-Partition': 1,
                 'X-Account-Device': 'sdb'},
                {'X-Account-Host': '10.0.0.2:1002',
                 'X-Account-Partition': 1,
                 'X-Account-Device': 'sdc'}])


class TestAccountController(unittest.TestCase):

    def setUp(self):
        self.app = proxy_server.Application(None, FakeMemcache(),
                                            account_ring=FakeRing(),
                                            container_ring=FakeRing(),
                                            object_ring=FakeRing)

    def assert_status_map(self, method, statuses, expected):
        with save_globals():
            set_http_connect(*statuses)
            req = Request.blank('/a', {})
            self.app.update_request(req)
            res = method(req)
            self.assertEquals(res.status_int, expected)
            set_http_connect(*statuses)
            req = Request.blank('/a/', {})
            self.app.update_request(req)
            res = method(req)
            self.assertEquals(res.status_int, expected)

    def test_OPTIONS(self):
        with save_globals():
            self.app.allow_account_management = False
            controller = proxy_server.AccountController(self.app, 'account')
            req = Request.blank('/account', {'REQUEST_METHOD': 'OPTIONS'})
            req.content_length = 0
            resp = controller.OPTIONS(req)
            self.assertEquals(200, resp.status_int)
            for verb in 'OPTIONS GET POST HEAD'.split():
                self.assertTrue(
                    verb in resp.headers['Allow'])
            self.assertEquals(len(resp.headers['Allow'].split(', ')), 4)

            # Test a CORS OPTIONS request (i.e. including Origin and
            # Access-Control-Request-Method headers)
            self.app.allow_account_management = False
            controller = proxy_server.AccountController(self.app, 'account')
            req = Request.blank('/account', {'REQUEST_METHOD': 'OPTIONS'},
                headers={'Origin': 'http://foo.com',
                                     'Access-Control-Request-Method': 'GET'})
            req.content_length = 0
            resp = controller.OPTIONS(req)
            self.assertEquals(200, resp.status_int)
            for verb in 'OPTIONS GET POST HEAD'.split():
                self.assertTrue(
                    verb in resp.headers['Allow'])
            self.assertEquals(len(resp.headers['Allow'].split(', ')), 4)

            self.app.allow_account_management = True
            controller = proxy_server.AccountController(self.app, 'account')
            req = Request.blank('/account', {'REQUEST_METHOD': 'OPTIONS'})
            req.content_length = 0
            resp = controller.OPTIONS(req)
            self.assertEquals(200, resp.status_int)
            print resp.headers['Allow']
            for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split():
                self.assertTrue(
                    verb in resp.headers['Allow'])
            self.assertEquals(len(resp.headers['Allow'].split(', ')), 6)

    def test_GET(self):
        with save_globals():
            controller = proxy_server.AccountController(self.app, 'account')
            self.assert_status_map(controller.GET, (200, 200, 200), 200)
            self.assert_status_map(controller.GET, (200, 200, 503), 200)
            self.assert_status_map(controller.GET, (200, 503, 503), 200)
            self.assert_status_map(controller.GET, (204, 204, 204), 204)
            self.assert_status_map(controller.GET, (204, 204, 503), 204)
            self.assert_status_map(controller.GET, (204, 503, 503), 204)
            self.assert_status_map(controller.GET, (204, 204, 200), 204)
            self.assert_status_map(controller.GET, (204, 200, 200), 204)
            self.assert_status_map(controller.GET, (404, 404, 404), 404)
            self.assert_status_map(controller.GET, (404, 404, 200), 200)
            self.assert_status_map(controller.GET, (404, 200, 200), 200)
            self.assert_status_map(controller.GET, (404, 404, 503), 404)
            self.assert_status_map(controller.GET, (404, 503, 503), 503)
            self.assert_status_map(controller.GET, (404, 204, 503), 204)

            self.app.memcache = FakeMemcacheReturnsNone()
            self.assert_status_map(controller.GET, (404, 404, 404), 404)

    def test_GET_autocreate(self):
        with save_globals():
            controller = proxy_server.AccountController(self.app, 'account')
            self.app.memcache = FakeMemcacheReturnsNone()
            self.assert_status_map(controller.GET,
                                   (404, 404, 404, 201, 201, 201, 204), 404)
            controller.app.account_autocreate = True
            self.assert_status_map(controller.GET,
                                   (404, 404, 404, 201, 201, 201, 204), 204)
            self.assert_status_map(controller.GET,
                                   (404, 404, 404, 403, 403, 403, 403), 403)
            self.assert_status_map(controller.GET,
                                   (404, 404, 404, 409, 409, 409, 409), 409)

    def test_HEAD(self):
        with save_globals():
            controller = proxy_server.AccountController(self.app, 'account')
            self.assert_status_map(controller.HEAD, (200, 200, 200), 200)
            self.assert_status_map(controller.HEAD, (200, 200, 503), 200)
            self.assert_status_map(controller.HEAD, (200, 503, 503), 200)
            self.assert_status_map(controller.HEAD, (204, 204, 204), 204)
            self.assert_status_map(controller.HEAD, (204, 204, 503), 204)
            self.assert_status_map(controller.HEAD, (204, 503, 503), 204)
            self.assert_status_map(controller.HEAD, (204, 204, 200), 204)
            self.assert_status_map(controller.HEAD, (204, 200, 200), 204)
            self.assert_status_map(controller.HEAD, (404, 404, 404), 404)
            self.assert_status_map(controller.HEAD, (404, 404, 200), 200)
            self.assert_status_map(controller.HEAD, (404, 200, 200), 200)
            self.assert_status_map(controller.HEAD, (404, 404, 503), 404)
            self.assert_status_map(controller.HEAD, (404, 503, 503), 503)
            self.assert_status_map(controller.HEAD, (404, 204, 503), 204)

    def test_HEAD_autocreate(self):
        with save_globals():
            controller = proxy_server.AccountController(self.app, 'account')
            self.app.memcache = FakeMemcacheReturnsNone()
            self.assert_status_map(controller.HEAD,
                                   (404, 404, 404, 201, 201, 201, 204), 404)
            controller.app.account_autocreate = True
            self.assert_status_map(controller.HEAD,
                                   (404, 404, 404, 201, 201, 201, 204), 204)
            self.assert_status_map(controller.HEAD,
                                   (404, 404, 404, 403, 403, 403, 403), 403)
            self.assert_status_map(controller.HEAD,
                                   (404, 404, 404, 409, 409, 409, 409), 409)

    def test_POST_autocreate(self):
        with save_globals():
            controller = proxy_server.AccountController(self.app, 'account')
            self.app.memcache = FakeMemcacheReturnsNone()
            self.assert_status_map(controller.POST,
                                   (404, 404, 404, 201, 201, 201), 404)
            controller.app.account_autocreate = True
            self.assert_status_map(controller.POST,
                                   (404, 404, 404, 201, 201, 201), 201)
            self.assert_status_map(controller.POST,
                                   (404, 404, 404, 403, 403, 403, 403), 403)
            self.assert_status_map(controller.POST,
                                   (404, 404, 404, 409, 409, 409, 409), 409)

    def test_connection_refused(self):
        self.app.account_ring.get_nodes('account')
        for dev in self.app.account_ring.devs.values():
            dev['ip'] = '127.0.0.1'
            dev['port'] = 1  # can't connect on this port
        controller = proxy_server.AccountController(self.app, 'account')
        req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'})
        self.app.update_request(req)
        resp = controller.HEAD(req)
        self.assertEquals(resp.status_int, 503)

    def test_other_socket_error(self):
        self.app.account_ring.get_nodes('account')
        for dev in self.app.account_ring.devs.values():
            dev['ip'] = '127.0.0.1'
            dev['port'] = -1  # invalid port number
        controller = proxy_server.AccountController(self.app, 'account')
        req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'})
        self.app.update_request(req)
        resp = controller.HEAD(req)
        self.assertEquals(resp.status_int, 503)

    def test_response_get_accept_ranges_header(self):
        with save_globals():
            set_http_connect(200, 200, body='{}')
            controller = proxy_server.AccountController(self.app, 'account')
            req = Request.blank('/a?format=json')
            self.app.update_request(req)
            res = controller.GET(req)
            self.assert_('accept-ranges' in res.headers)
            self.assertEqual(res.headers['accept-ranges'], 'bytes')

    def test_response_head_accept_ranges_header(self):
        with save_globals():
            set_http_connect(200, 200, body='{}')
            controller = proxy_server.AccountController(self.app, 'account')
            req = Request.blank('/a?format=json')
            self.app.update_request(req)
            res = controller.HEAD(req)
            res.body
            self.assert_('accept-ranges' in res.headers)
            self.assertEqual(res.headers['accept-ranges'], 'bytes')

    def test_PUT(self):
        with save_globals():
            controller = proxy_server.AccountController(self.app, 'account')

            def test_status_map(statuses, expected, **kwargs):
                set_http_connect(*statuses, **kwargs)
                self.app.memcache.store = {}
                req = Request.blank('/a', {})
                req.content_length = 0
                self.app.update_request(req)
                res = controller.PUT(req)
                expected = str(expected)
                self.assertEquals(res.status[:len(expected)], expected)
            test_status_map((201, 201, 201), 405)
            self.app.allow_account_management = True
            test_status_map((201, 201, 201), 201)
            test_status_map((201, 201, 500), 201)
            test_status_map((201, 500, 500), 503)
            test_status_map((204, 500, 404), 503)

    def test_PUT_max_account_name_length(self):
        with save_globals():
            self.app.allow_account_management = True
            limit = MAX_ACCOUNT_NAME_LENGTH
            controller = proxy_server.AccountController(self.app, '1' * limit)
            self.assert_status_map(controller.PUT, (201, 201, 201), 201)
            controller = proxy_server.AccountController(
                self.app, '2' * (limit + 1))
            self.assert_status_map(controller.PUT, (201, 201, 201), 400)

    def test_PUT_connect_exceptions(self):
        with save_globals():
            self.app.allow_account_management = True
            controller = proxy_server.AccountController(self.app, 'account')
            self.assert_status_map(controller.PUT, (201, 201, -1), 201)
            self.assert_status_map(controller.PUT, (201, -1, -1), 503)
            self.assert_status_map(controller.PUT, (503, 503, -1), 503)

    def test_PUT_metadata(self):
        self.metadata_helper('PUT')

    def test_POST_metadata(self):
        self.metadata_helper('POST')

    def metadata_helper(self, method):
        for test_header, test_value in (
                ('X-Account-Meta-TestHeader', 'TestValue'),
                ('X-Account-Meta-TestHeader', ''),
                ('X-Remove-Account-Meta-TestHeader', 'anything')):
            test_errors = []

            def test_connect(ipaddr, port, device, partition, method, path,
                             headers=None, query_string=None):
                if path == '/a':
                    find_header = test_header
                    find_value = test_value
                    if find_header.lower().startswith('x-remove-'):
                        find_header = \
                            find_header.lower().replace('-remove', '', 1)
                        find_value = ''
                    for k, v in headers.iteritems():
                        if k.lower() == find_header.lower() and \
                                v == find_value:
                            break
                    else:
                        test_errors.append('%s: %s not in %s' %
                                           (find_header, find_value, headers))
            with save_globals():
                self.app.allow_account_management = True
                controller = \
                    proxy_server.AccountController(self.app, 'a')
                set_http_connect(201, 201, 201, give_connect=test_connect)
                req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                    headers={test_header: test_value})
                self.app.update_request(req)
                res = getattr(controller, method)(req)
                self.assertEquals(test_errors, [])

    def test_PUT_bad_metadata(self):
        self.bad_metadata_helper('PUT')

    def test_POST_bad_metadata(self):
        self.bad_metadata_helper('POST')

    def bad_metadata_helper(self, method):
        with save_globals():
            self.app.allow_account_management = True
            controller = proxy_server.AccountController(self.app, 'a')
            set_http_connect(200, 201, 201, 201)
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method})
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 201)

            set_http_connect(201, 201, 201)
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers={'X-Account-Meta-' +
                                ('a' * MAX_META_NAME_LENGTH): 'v'})
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 201)
            set_http_connect(201, 201, 201)
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers={'X-Account-Meta-' +
                                ('a' * (MAX_META_NAME_LENGTH + 1)): 'v'})
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 400)

            set_http_connect(201, 201, 201)
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers={'X-Account-Meta-Too-Long':
                                'a' * MAX_META_VALUE_LENGTH})
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 201)
            set_http_connect(201, 201, 201)
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers={'X-Account-Meta-Too-Long':
                                'a' * (MAX_META_VALUE_LENGTH + 1)})
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 400)

            set_http_connect(201, 201, 201)
            headers = {}
            for x in xrange(MAX_META_COUNT):
                headers['X-Account-Meta-%d' % x] = 'v'
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers=headers)
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 201)
            set_http_connect(201, 201, 201)
            headers = {}
            for x in xrange(MAX_META_COUNT + 1):
                headers['X-Account-Meta-%d' % x] = 'v'
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers=headers)
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 400)

            set_http_connect(201, 201, 201)
            headers = {}
            header_value = 'a' * MAX_META_VALUE_LENGTH
            size = 0
            x = 0
            while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH:
                size += 4 + MAX_META_VALUE_LENGTH
                headers['X-Account-Meta-%04d' % x] = header_value
                x += 1
            if MAX_META_OVERALL_SIZE - size > 1:
                headers['X-Account-Meta-a'] = \
                    'a' * (MAX_META_OVERALL_SIZE - size - 1)
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers=headers)
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 201)
            set_http_connect(201, 201, 201)
            headers['X-Account-Meta-a'] = \
                'a' * (MAX_META_OVERALL_SIZE - size)
            req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                headers=headers)
            self.app.update_request(req)
            resp = getattr(controller, method)(req)
            self.assertEquals(resp.status_int, 400)

    def test_DELETE(self):
        with save_globals():
            controller = proxy_server.AccountController(self.app, 'account')

            def test_status_map(statuses, expected, **kwargs):
                set_http_connect(*statuses, **kwargs)
                self.app.memcache.store = {}
                req = Request.blank('/a', {'REQUEST_METHOD': 'DELETE'})
                req.content_length = 0
                self.app.update_request(req)
                res = controller.DELETE(req)
                expected = str(expected)
                self.assertEquals(res.status[:len(expected)], expected)
            test_status_map((201, 201, 201), 405)
            self.app.allow_account_management = True
            test_status_map((201, 201, 201), 201)
            test_status_map((201, 201, 500), 201)
            test_status_map((201, 500, 500), 503)
            test_status_map((204, 500, 404), 503)

    def test_DELETE_with_query_string(self):
        # Extra safety in case someone typos a query string for an
        # account-level DELETE request that was really meant to be caught by
        # some middleware.
        with save_globals():
            controller = proxy_server.AccountController(self.app, 'account')

            def test_status_map(statuses, expected, **kwargs):
                set_http_connect(*statuses, **kwargs)
                self.app.memcache.store = {}
                req = Request.blank('/a?whoops', {'REQUEST_METHOD': 'DELETE'})
                req.content_length = 0
                self.app.update_request(req)
                res = controller.DELETE(req)
                expected = str(expected)
                self.assertEquals(res.status[:len(expected)], expected)
            test_status_map((201, 201, 201), 400)
            self.app.allow_account_management = True
            test_status_map((201, 201, 201), 400)
            test_status_map((201, 201, 500), 400)
            test_status_map((201, 500, 500), 400)
            test_status_map((204, 500, 404), 400)


class FakeObjectController(object):

    def __init__(self):
        self.app = self
        self.logger = self
        self.account_name = 'a'
        self.container_name = 'c'
        self.object_name = 'o'
        self.trans_id = 'tx1'
        self.object_ring = FakeRing()
        self.node_timeout = 1
        self.rate_limit_after_segment = 3
        self.rate_limit_segments_per_sec = 2

    def exception(self, *args):
        self.exception_args = args
        self.exception_info = sys.exc_info()

    def GETorHEAD_base(self, *args):
        self.GETorHEAD_base_args = args
        req = args[0]
        path = args[4]
        body = data = path[-1] * int(path[-1])
        if req.range:
            r = req.range.ranges_for_length(len(data))
            if r:
                (start, stop) = r[0]
                body = data[start:stop]
        resp = Response(app_iter=iter(body))
        return resp

    def iter_nodes(self, ring, partition):
        for node in ring.get_part_nodes(partition):
            yield node
        for node in ring.get_more_nodes(partition):
            yield node

    def sort_nodes(self, nodes):
        return nodes

    def set_node_timing(self, node, timing):
        return


class Stub(object):
    pass


class TestSegmentedIterable(unittest.TestCase):

    def setUp(self):
        self.controller = FakeObjectController()

    def test_load_next_segment_unexpected_error(self):
        # Iterator value isn't a dict
        self.assertRaises(Exception,
                          SegmentedIterable(self.controller, None,
                                            [None])._load_next_segment)
        self.assert_(self.controller.exception_args[0].startswith(
                     'ERROR: While processing manifest'))

    def test_load_next_segment_with_no_segments(self):
        self.assertRaises(StopIteration,
                          SegmentedIterable(self.controller, 'lc',
                                            [])._load_next_segment)

    def test_load_next_segment_with_one_segment(self):
        segit = SegmentedIterable(self.controller, 'lc', [{'name':
                                  'o1'}])
        segit._load_next_segment()
        self.assertEquals(self.controller.GETorHEAD_base_args[4], '/a/lc/o1')
        data = ''.join(segit.segment_iter)
        self.assertEquals(data, '1')

    def test_load_next_segment_with_two_segments(self):
        segit = SegmentedIterable(self.controller, 'lc', [{'name':
                                  'o1'}, {'name': 'o2'}])
        segit._load_next_segment()
        self.assertEquals(self.controller.GETorHEAD_base_args[4], '/a/lc/o1')
        data = ''.join(segit.segment_iter)
        self.assertEquals(data, '1')
        segit._load_next_segment()
        self.assertEquals(self.controller.GETorHEAD_base_args[4], '/a/lc/o2')
        data = ''.join(segit.segment_iter)
        self.assertEquals(data, '22')

    def test_load_next_segment_rate_limiting(self):
        sleep_calls = []

        def _stub_sleep(sleepy_time):
            sleep_calls.append(sleepy_time)
        orig_sleep = swift.proxy.controllers.obj.sleep
        try:
            swift.proxy.controllers.obj.sleep = _stub_sleep
            segit = SegmentedIterable(
                self.controller, 'lc', [
                    {'name': 'o1'}, {'name': 'o2'}, {'name': 'o3'},
                    {'name': 'o4'}, {'name': 'o5'}])

            # rate_limit_after_segment == 3, so the first 3 segments should
            # invoke no sleeping.
            for _ in xrange(3):
                segit._load_next_segment()
            self.assertEquals([], sleep_calls)
            self.assertEquals(self.controller.GETorHEAD_base_args[4],
                              '/a/lc/o3')

            # Loading of next (4th) segment starts rate-limiting.
            segit._load_next_segment()
            self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
            self.assertEquals(self.controller.GETorHEAD_base_args[4],
                              '/a/lc/o4')

            sleep_calls = []
            segit._load_next_segment()
            self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
            self.assertEquals(self.controller.GETorHEAD_base_args[4],
                              '/a/lc/o5')
        finally:
            swift.proxy.controllers.obj.sleep = orig_sleep

    def test_load_next_segment_range_req_rate_limiting(self):
        sleep_calls = []

        def _stub_sleep(sleepy_time):
            sleep_calls.append(sleepy_time)
        orig_sleep = swift.proxy.controllers.obj.sleep
        try:
            swift.proxy.controllers.obj.sleep = _stub_sleep
            segit = SegmentedIterable(
                self.controller, 'lc', [
                    {'name': 'o0', 'bytes': 5}, {'name': 'o1', 'bytes': 5},
                    {'name': 'o2', 'bytes': 1}, {'name': 'o3'}, {'name': 'o4'},
                    {'name': 'o5'}, {'name': 'o6'}])

            # this tests for a range request which skips over the whole first
            # segment, after that 3 segments will be read in because the
            # rate_limit_after_segment == 3, then sleeping starts
            segit_iter = segit.app_iter_range(10, None)
            segit_iter.next()
            for _ in xrange(2):
                # this is set to 2 instead of 3 because o2 was loaded after
                # o0 and o1 were skipped.
                segit._load_next_segment()
            self.assertEquals([], sleep_calls)
            self.assertEquals(self.controller.GETorHEAD_base_args[4],
                              '/a/lc/o4')

            # Loading of next (5th) segment starts rate-limiting.
            segit._load_next_segment()
            self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
            self.assertEquals(self.controller.GETorHEAD_base_args[4],
                              '/a/lc/o5')

            sleep_calls = []
            segit._load_next_segment()
            self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
            self.assertEquals(self.controller.GETorHEAD_base_args[4],
                              '/a/lc/o6')
        finally:
            swift.proxy.controllers.obj.sleep = orig_sleep

    def test_load_next_segment_with_two_segments_skip_first(self):
        segit = SegmentedIterable(self.controller, 'lc', [{'name':
                                  'o1'}, {'name': 'o2'}])
        segit.ratelimit_index = 0
        segit.listing.next()
        segit._load_next_segment()
        self.assertEquals(self.controller.GETorHEAD_base_args[4], '/a/lc/o2')
        data = ''.join(segit.segment_iter)
        self.assertEquals(data, '22')

    def test_load_next_segment_with_seek(self):
        segit = SegmentedIterable(self.controller, 'lc', [{'name':
                                  'o1'}, {'name': 'o2'}])
        segit.ratelimit_index = 0
        segit.listing.next()
        segit.seek = 1
        segit._load_next_segment()
        self.assertEquals(self.controller.GETorHEAD_base_args[4], '/a/lc/o2')
        self.assertEquals(str(self.controller.GETorHEAD_base_args[0].range),
                          'bytes=1-')
        data = ''.join(segit.segment_iter)
        self.assertEquals(data, '2')

    def test_load_next_segment_with_get_error(self):

        def local_GETorHEAD_base(*args):
            return HTTPNotFound()

        self.controller.GETorHEAD_base = local_GETorHEAD_base
        self.assertRaises(Exception,
                          SegmentedIterable(self.controller, 'lc',
                          [{'name': 'o1'}])._load_next_segment)
        self.assert_(self.controller.exception_args[0].startswith(
                     'ERROR: While processing manifest'))
        self.assertEquals(str(self.controller.exception_info[1]),
                          'Could not load object segment /a/lc/o1: 404')

    def test_iter_unexpected_error(self):
        # Iterator value isn't a dict
        self.assertRaises(Exception, ''.join,
                          SegmentedIterable(self.controller, None, [None]))
        self.assert_(self.controller.exception_args[0].startswith(
            'ERROR: While processing manifest'))

    def test_iter_with_no_segments(self):
        segit = SegmentedIterable(self.controller, 'lc', [])
        self.assertEquals(''.join(segit), '')

    def test_iter_with_one_segment(self):
        segit = SegmentedIterable(self.controller, 'lc', [{'name':
                                  'o1'}])
        segit.response = Stub()
        self.assertEquals(''.join(segit), '1')

    def test_iter_with_two_segments(self):
        segit = SegmentedIterable(self.controller, 'lc', [{'name':
                                  'o1'}, {'name': 'o2'}])
        segit.response = Stub()
        self.assertEquals(''.join(segit), '122')

    def test_iter_with_get_error(self):

        def local_GETorHEAD_base(*args):
            return HTTPNotFound()

        self.controller.GETorHEAD_base = local_GETorHEAD_base
        self.assertRaises(Exception, ''.join,
                          SegmentedIterable(self.controller, 'lc', [{'name':
                                                                    'o1'}]))
        self.assert_(self.controller.exception_args[0].startswith(
                     'ERROR: While processing manifest'))
        self.assertEquals(str(self.controller.exception_info[1]),
                          'Could not load object segment /a/lc/o1: 404')

    def test_app_iter_range_unexpected_error(self):
        # Iterator value isn't a dict
        self.assertRaises(Exception,
                          SegmentedIterable(self.controller, None,
                                            [None]).app_iter_range(None,
                                                                   None).next)
        self.assert_(self.controller.exception_args[0].startswith(
            'ERROR: While processing manifest'))

    def test_app_iter_range_with_no_segments(self):
        self.assertEquals(''.join(SegmentedIterable(
            self.controller, 'lc', []).app_iter_range(None, None)), '')
        self.assertEquals(''.join(SegmentedIterable(
            self.controller, 'lc', []).app_iter_range(3, None)), '')
        self.assertEquals(''.join(SegmentedIterable(
            self.controller, 'lc', []).app_iter_range(3, 5)), '')
        self.assertEquals(''.join(SegmentedIterable(
            self.controller, 'lc', []).app_iter_range(None, 5)), '')

    def test_app_iter_range_with_one_segment(self):
        listing = [{'name': 'o1', 'bytes': 1}]

        segit = SegmentedIterable(self.controller, 'lc', listing)
        segit.response = Stub()
        self.assertEquals(''.join(segit.app_iter_range(None, None)), '1')

        segit = SegmentedIterable(self.controller, 'lc', listing)
        self.assertEquals(''.join(segit.app_iter_range(3, None)), '')

        segit = SegmentedIterable(self.controller, 'lc', listing)
        self.assertEquals(''.join(segit.app_iter_range(3, 5)), '')

        segit = SegmentedIterable(self.controller, 'lc', listing)
        segit.response = Stub()
        self.assertEquals(''.join(segit.app_iter_range(None, 5)), '1')

    def test_app_iter_range_with_two_segments(self):
        listing = [{'name': 'o1', 'bytes': 1}, {'name': 'o2', 'bytes': 2}]

        segit = SegmentedIterable(self.controller, 'lc', listing)
        segit.response = Stub()
        self.assertEquals(''.join(segit.app_iter_range(None, None)), '122')

        segit = SegmentedIterable(self.controller, 'lc', listing)
        segit.response = Stub()
        self.assertEquals(''.join(segit.app_iter_range(1, None)), '22')

        segit = SegmentedIterable(self.controller, 'lc', listing)
        segit.response = Stub()
        self.assertEquals(''.join(segit.app_iter_range(1, 5)), '22')

        segit = SegmentedIterable(self.controller, 'lc', listing)
        segit.response = Stub()
        self.assertEquals(''.join(segit.app_iter_range(None, 2)), '12')

    def test_app_iter_range_with_many_segments(self):
        listing = [{'name': 'o1', 'bytes': 1}, {'name': 'o2', 'bytes': 2},
                   {'name': 'o3', 'bytes': 3}, {'name': 'o4', 'bytes': 4},
                   {'name': 'o5', 'bytes': 5}]

        segit = SegmentedIterable(self.controller, 'lc', listing)
        segit.response = Stub()
        self.assertEquals(''.join(segit.app_iter_range(None, None)),
                          '122333444455555')

        segit = SegmentedIterable(self.controller, 'lc', listing)
        segit.response = Stub()
        self.assertEquals(''.join(segit.app_iter_range(3, None)),
                          '333444455555')

        segit = SegmentedIterable(self.controller, 'lc', listing)
        segit.response = Stub()
        self.assertEquals(''.join(segit.app_iter_range(5, None)), '3444455555')

        segit = SegmentedIterable(self.controller, 'lc', listing)
        segit.response = Stub()
        self.assertEquals(''.join(segit.app_iter_range(None, 6)), '122333')

        segit = SegmentedIterable(self.controller, 'lc', listing)
        segit.response = Stub()
        self.assertEquals(''.join(segit.app_iter_range(None, 7)), '1223334')

        segit = SegmentedIterable(self.controller, 'lc', listing)
        segit.response = Stub()
        self.assertEquals(''.join(segit.app_iter_range(3, 7)), '3334')

        segit = SegmentedIterable(self.controller, 'lc', listing)
        segit.response = Stub()
        self.assertEquals(''.join(segit.app_iter_range(5, 7)), '34')


if __name__ == '__main__':
    setup()
    try:
        unittest.main()
    finally:
        teardown()