Generic means for persisting system metadata.

Middleware or core features may need to store metadata
against accounts or containers. This patch adds a
generic mechanism for system metadata to be persisted
in backend databases, without polluting the user
metadata namespace, by using the reserved header
namespace x-<server_type>-sysmeta-*.

Modifications are firstly that backend servers persist
system metadata headers alongside user metadata and
other system state.

For accounts and containers, system metadata in PUT
and POST requests is treated in a similar way to user
metadata. System metadata is not yet supported for
object requests.

Secondly, changes in the proxy controllers ensure that
headers in the system metadata namespace will pass through
in requests to backend servers.

Thirdly, system metadata returned from backend servers
in GET or HEAD responses is added to the cached info
dict, which middleware can access.

Finally, a gatekeeper middleware module is provided
which filters all system metadata headers from requests
and responses by removing headers with names starting
x-account-sysmeta-, x-container-sysmeta-. The gatekeeper
also removes headers starting x-object-sysmeta- in
anticipation of future support for system metadata being
set for objects. This prevents clients from writing or
reading system metadata.

The required_filters list in swift/proxy/server.py is
modified to include the gatekeeper middleware so that
if the gatekeeper has not been configured in the
pipeline then it will be automatically inserted close
to the start of the pipeline.

blueprint cluster-federation

Change-Id: I80b8b14243cc59505f8c584920f8f527646b5f45

etc/proxy-server.conf-sample

@@ -69,7 +69,7 @@
 # eventlet_debug = false
 
 [pipeline:main]
-pipeline = catch_errors healthcheck proxy-logging cache bulk slo ratelimit tempauth container-quotas account-quotas proxy-logging proxy-server
+pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk slo ratelimit tempauth container-quotas account-quotas proxy-logging proxy-server
 
 [app:proxy-server]
 use = egg:swift#proxy
@@ -509,3 +509,12 @@ use = egg:swift#slo
 
 [filter:account-quotas]
 use = egg:swift#account_quotas
+
+[filter:gatekeeper]
+use = egg:swift#gatekeeper
+# You can override the default log routing for this filter here:
+# set log_name = gatekeeper
+# set log_facility = LOG_LOCAL0
+# set log_level = INFO
+# set log_headers = false
+# set log_address = /dev/log

setup.cfg

@@ -86,6 +86,7 @@ paste.filter_factory =
     proxy_logging = swift.common.middleware.proxy_logging:filter_factory
     slo = swift.common.middleware.slo:filter_factory
     list_endpoints = swift.common.middleware.list_endpoints:filter_factory
+    gatekeeper = swift.common.middleware.gatekeeper:filter_factory
 
 [build_sphinx]
 all_files = 1

swift/account/server.py

@@ -37,6 +37,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, \
     HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, \
     HTTPPreconditionFailed, HTTPConflict, Request, \
     HTTPInsufficientStorage, HTTPException
+from swift.common.request_helpers import is_sys_or_user_meta
 
 
 DATADIR = 'accounts'
@@ -152,7 +153,7 @@ class AccountController(object):
             metadata = {}
             metadata.update((key, (value, timestamp))
                             for key, value in req.headers.iteritems()
-                            if key.lower().startswith('x-account-meta-'))
+                            if is_sys_or_user_meta('account', key))
             if metadata:
                 broker.update_metadata(metadata)
             if created:
@@ -258,7 +259,7 @@ class AccountController(object):
         metadata = {}
         metadata.update((key, (value, timestamp))
                         for key, value in req.headers.iteritems()
-                        if key.lower().startswith('x-account-meta-'))
+                        if is_sys_or_user_meta('account', key))
         if metadata:
             broker.update_metadata(metadata)
         return HTTPNoContent(request=req)

swift/common/middleware/gatekeeper.py

@@ -0,0 +1,94 @@
+# Copyright (c) 2010-2012 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+The ``gatekeeper`` middleware imposes restrictions on the headers that
+may be included with requests and responses. Request headers are filtered
+to remove headers that should never be generated by a client. Similarly,
+response headers are filtered to remove private headers that should
+never be passed to a client.
+
+The ``gatekeeper`` middleware must always be present in the proxy server
+wsgi pipeline. It should be configured close to the start of the pipeline
+specified in ``/etc/swift/proxy-server.conf``, immediately after catch_errors
+and before any other middleware. It is essential that it is configured ahead
+of all middlewares using system metadata in order that they function
+correctly.
+
+If ``gatekeeper`` middleware is not configured in the pipeline then it will be
+automatically inserted close to the start of the pipeline by the proxy server.
+"""
+
+
+from swift.common.swob import wsgify
+from swift.common.utils import get_logger
+from swift.common.request_helpers import remove_items, get_sys_meta_prefix
+import re
+
+"""
+A list of python regular expressions that will be used to
+match against inbound request headers. Matching headers will
+be removed from the request.
+"""
+# Exclude headers starting with a sysmeta prefix.
+# If adding to this list, note that these are regex patterns,
+# so use a trailing $ to constrain to an exact header match
+# rather than prefix match.
+inbound_exclusions = [get_sys_meta_prefix('account'),
+                      get_sys_meta_prefix('container'),
+                      get_sys_meta_prefix('object')]
+# 'x-object-sysmeta' is reserved in anticipation of future support
+# for system metadata being applied to objects
+
+
+"""
+A list of python regular expressions that will be used to
+match against outbound response headers. Matching headers will
+be removed from the response.
+"""
+outbound_exclusions = inbound_exclusions
+
+
+def make_exclusion_test(exclusions):
+    expr = '|'.join(exclusions)
+    test = re.compile(expr, re.IGNORECASE)
+    return test.match
+
+
+class GatekeeperMiddleware(object):
+    def __init__(self, app, conf):
+        self.app = app
+        self.logger = get_logger(conf, log_route='gatekeeper')
+        self.inbound_condition = make_exclusion_test(inbound_exclusions)
+        self.outbound_condition = make_exclusion_test(outbound_exclusions)
+
+    @wsgify
+    def __call__(self, req):
+        removed = remove_items(req.headers, self.inbound_condition)
+        if removed:
+            self.logger.debug('removed request headers: %s' % removed)
+        resp = req.get_response(self.app)
+        removed = remove_items(resp.headers, self.outbound_condition)
+        if removed:
+            self.logger.debug('removed response headers: %s' % removed)
+        return resp
+
+
+def filter_factory(global_conf, **local_conf):
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    def gatekeeper_filter(app):
+        return GatekeeperMiddleware(app, conf)
+    return gatekeeper_filter

swift/common/request_helpers.py

@@ -87,3 +87,109 @@ def split_and_validate_path(request, minsegs=1, maxsegs=None,
     except ValueError as err:
         raise HTTPBadRequest(body=str(err), request=request,
                              content_type='text/plain')
+
+
+def is_user_meta(server_type, key):
+    """
+    Tests if a header key starts with and is longer than the user
+    metadata prefix for given server type.
+
+    :param server_type: type of backend server i.e. [account|container|object]
+    :param key: header key
+    :returns: True if the key satisfies the test, False otherwise
+    """
+    if len(key) <= 8 + len(server_type):
+        return False
+    return key.lower().startswith(get_user_meta_prefix(server_type))
+
+
+def is_sys_meta(server_type, key):
+    """
+    Tests if a header key starts with and is longer than the system
+    metadata prefix for given server type.
+
+    :param server_type: type of backend server i.e. [account|container|object]
+    :param key: header key
+    :returns: True if the key satisfies the test, False otherwise
+    """
+    if len(key) <= 11 + len(server_type):
+        return False
+    return key.lower().startswith(get_sys_meta_prefix(server_type))
+
+
+def is_sys_or_user_meta(server_type, key):
+    """
+    Tests if a header key starts with and is longer than the user or system
+    metadata prefix for given server type.
+
+    :param server_type: type of backend server i.e. [account|container|object]
+    :param key: header key
+    :returns: True if the key satisfies the test, False otherwise
+    """
+    return is_user_meta(server_type, key) or is_sys_meta(server_type, key)
+
+
+def strip_user_meta_prefix(server_type, key):
+    """
+    Removes the user metadata prefix for a given server type from the start
+    of a header key.
+
+    :param server_type: type of backend server i.e. [account|container|object]
+    :param key: header key
+    :returns: stripped header key
+    """
+    return key[len(get_user_meta_prefix(server_type)):]
+
+
+def strip_sys_meta_prefix(server_type, key):
+    """
+    Removes the system metadata prefix for a given server type from the start
+    of a header key.
+
+    :param server_type: type of backend server i.e. [account|container|object]
+    :param key: header key
+    :returns: stripped header key
+    """
+    return key[len(get_sys_meta_prefix(server_type)):]
+
+
+def get_user_meta_prefix(server_type):
+    """
+    Returns the prefix for user metadata headers for given server type.
+
+    This prefix defines the namespace for headers that will be persisted
+    by backend servers.
+
+    :param server_type: type of backend server i.e. [account|container|object]
+    :returns: prefix string for server type's user metadata headers
+    """
+    return 'x-%s-%s-' % (server_type.lower(), 'meta')
+
+
+def get_sys_meta_prefix(server_type):
+    """
+    Returns the prefix for system metadata headers for given server type.
+
+    This prefix defines the namespace for headers that will be persisted
+    by backend servers.
+
+    :param server_type: type of backend server i.e. [account|container|object]
+    :returns: prefix string for server type's system metadata headers
+    """
+    return 'x-%s-%s-' % (server_type.lower(), 'sysmeta')
+
+
+def remove_items(headers, condition):
+    """
+    Removes items from a dict whose keys satisfy
+    the given condition.
+
+    :param headers: a dict of headers
+    :param condition: a function that will be passed the header key as a
+    single argument and should return True if the header is to be removed.
+    :returns: a dict, possibly empty, of headers that have been removed
+    """
+    removed = {}
+    keys = filter(condition, headers)
+    removed.update((key, headers.pop(key)) for key in keys)
+    return removed

swift/common/wsgi.py

@@ -213,6 +213,21 @@ class PipelineWrapper(object):
         except ValueError:
             return False
 
+    def startswith(self, entry_point_name):
+        """
+        Tests if the pipeline starts with the given entry point name.
+
+        :param entry_point_name: entry point of middleware or app (Swift only)
+
+        :returns: True if entry_point_name is first in pipeline, False
+        otherwise
+        """
+        try:
+            first_ctx = self.context.filter_contexts[0]
+        except IndexError:
+            first_ctx = self.context.app_context
+        return first_ctx.entry_point_name == entry_point_name
+
     def _format_for_display(self, ctx):
         if ctx.entry_point_name:
             return ctx.entry_point_name

swift/container/server.py

@@ -26,7 +26,7 @@ import swift.common.db
 from swift.container.backend import ContainerBroker
 from swift.common.db import DatabaseAlreadyExists
 from swift.common.request_helpers import get_param, get_listing_content_type, \
-    split_and_validate_path
+    split_and_validate_path, is_sys_or_user_meta
 from swift.common.utils import get_logger, hash_path, public, \
     normalize_timestamp, storage_directory, validate_sync_to, \
     config_true_value, json, timing_stats, replication, \
@@ -266,7 +266,7 @@ class ContainerController(object):
                 (key, (value, timestamp))
                 for key, value in req.headers.iteritems()
                 if key.lower() in self.save_headers or
-                key.lower().startswith('x-container-meta-'))
+                is_sys_or_user_meta('container', key))
             if metadata:
                 if 'X-Container-Sync-To' in metadata:
                     if 'X-Container-Sync-To' not in broker.metadata or \
@@ -307,7 +307,7 @@ class ContainerController(object):
             (key, value)
             for key, (value, timestamp) in broker.metadata.iteritems()
             if value != '' and (key.lower() in self.save_headers or
-                                key.lower().startswith('x-container-meta-')))
+                                is_sys_or_user_meta('container', key)))
         headers['Content-Type'] = out_content_type
         return HTTPNoContent(request=req, headers=headers, charset='utf-8')
 
@@ -374,7 +374,7 @@ class ContainerController(object):
         }
         for key, (value, timestamp) in broker.metadata.iteritems():
             if value and (key.lower() in self.save_headers or
-                          key.lower().startswith('x-container-meta-')):
+                          is_sys_or_user_meta('container', key)):
                 resp_headers[key] = value
         ret = Response(request=req, headers=resp_headers,
                        content_type=out_content_type, charset='utf-8')
@@ -452,7 +452,7 @@ class ContainerController(object):
         metadata.update(
             (key, (value, timestamp)) for key, value in req.headers.iteritems()
             if key.lower() in self.save_headers or
-            key.lower().startswith('x-container-meta-'))
+            is_sys_or_user_meta('container', key))
         if metadata:
             if 'X-Container-Sync-To' in metadata:
                 if 'X-Container-Sync-To' not in broker.metadata or \

swift/obj/server.py

@@ -38,7 +38,7 @@ from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \
     DiskFileDeviceUnavailable, DiskFileExpired
 from swift.obj import ssync_receiver
 from swift.common.http import is_success
-from swift.common.request_helpers import split_and_validate_path
+from swift.common.request_helpers import split_and_validate_path, is_user_meta
 from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
     HTTPInternalServerError, HTTPNoContent, HTTPNotFound, HTTPNotModified, \
     HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
@@ -338,7 +338,7 @@ class ObjectController(object):
             return HTTPConflict(request=request)
         metadata = {'X-Timestamp': request.headers['x-timestamp']}
         metadata.update(val for val in request.headers.iteritems()
-                        if val[0].startswith('X-Object-Meta-'))
+                        if is_user_meta('object', val[0]))
         for header_key in self.allowed_headers:
             if header_key in request.headers:
                 header_caps = header_key.title()
@@ -422,8 +422,7 @@ class ObjectController(object):
                     'Content-Length': str(upload_size),
                 }
                 metadata.update(val for val in request.headers.iteritems()
-                                if val[0].lower().startswith('x-object-meta-')
-                                and len(val[0]) > 14)
+                                if is_user_meta('object', val[0]))
                 for header_key in (
                         request.headers.get('X-Backend-Replication-Headers') or
                         self.allowed_headers):
@@ -504,7 +503,7 @@ class ObjectController(object):
                 response.headers['Content-Type'] = metadata.get(
                     'Content-Type', 'application/octet-stream')
                 for key, value in metadata.iteritems():
-                    if key.lower().startswith('x-object-meta-') or \
+                    if is_user_meta('object', key) or \
                             key.lower() in self.allowed_headers:
                         response.headers[key] = value
                 response.etag = metadata['ETag']
@@ -545,7 +544,7 @@ class ObjectController(object):
         response.headers['Content-Type'] = metadata.get(
             'Content-Type', 'application/octet-stream')
         for key, value in metadata.iteritems():
-            if key.lower().startswith('x-object-meta-') or \
+            if is_user_meta('object', key) or \
                     key.lower() in self.allowed_headers:
                 response.headers[key] = value
         response.etag = metadata['ETag']

swift/proxy/controllers/base.py

@@ -48,6 +48,8 @@ from swift.common.http import is_informational, is_success, is_redirection, \
     HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED
 from swift.common.swob import Request, Response, HeaderKeyDict, Range, \
     HTTPException, HTTPRequestedRangeNotSatisfiable
+from swift.common.request_helpers import strip_sys_meta_prefix, \
+    strip_user_meta_prefix, is_user_meta, is_sys_meta, is_sys_or_user_meta
 
 
 def update_headers(response, headers):
@@ -106,11 +108,32 @@ def get_container_memcache_key(account, container):
     return cache_key
 
 
+def _prep_headers_to_info(headers, server_type):
+    """
+    Helper method that iterates once over a dict of headers,
+    converting all keys to lower case and separating
+    into subsets containing user metadata, system metadata
+    and other headers.
+    """
+    meta = {}
+    sysmeta = {}
+    other = {}
+    for key, val in dict(headers).iteritems():
+        lkey = key.lower()
+        if is_user_meta(server_type, lkey):
+            meta[strip_user_meta_prefix(server_type, lkey)] = val
+        elif is_sys_meta(server_type, lkey):
+            sysmeta[strip_sys_meta_prefix(server_type, lkey)] = val
+        else:
+            other[lkey] = val
+    return other, meta, sysmeta
+
+
 def headers_to_account_info(headers, status_int=HTTP_OK):
     """
     Construct a cacheable dict of account info based on response headers.
     """
-    headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
+    headers, meta, sysmeta = _prep_headers_to_info(headers, 'account')
     return {
         'status': status_int,
         # 'container_count' anomaly:
@@ -120,9 +143,8 @@ def headers_to_account_info(headers, status_int=HTTP_OK):
         'container_count': headers.get('x-account-container-count'),
         'total_object_count': headers.get('x-account-object-count'),
         'bytes': headers.get('x-account-bytes-used'),
-        'meta': dict((key[15:], value)
-                     for key, value in headers.iteritems()
-                     if key.startswith('x-account-meta-'))
+        'meta': meta,
+        'sysmeta': sysmeta
     }
 
 
@@ -130,7 +152,7 @@ def headers_to_container_info(headers, status_int=HTTP_OK):
     """
     Construct a cacheable dict of container info based on response headers.
     """
-    headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
+    headers, meta, sysmeta = _prep_headers_to_info(headers, 'container')
     return {
         'status': status_int,
         'read_acl': headers.get('x-container-read'),
@@ -140,16 +162,12 @@ def headers_to_container_info(headers, status_int=HTTP_OK):
         'bytes': headers.get('x-container-bytes-used'),
         'versions': headers.get('x-versions-location'),
         'cors': {
-            'allow_origin': headers.get(
-                'x-container-meta-access-control-allow-origin'),
-            'expose_headers': headers.get(
-                'x-container-meta-access-control-expose-headers'),
-            'max_age': headers.get(
-                'x-container-meta-access-control-max-age')
+            'allow_origin': meta.get('access-control-allow-origin'),
+            'expose_headers': meta.get('access-control-expose-headers'),
+            'max_age': meta.get('access-control-max-age')
         },
-        'meta': dict((key[17:], value)
-                     for key, value in headers.iteritems()
-                     if key.startswith('x-container-meta-'))
+        'meta': meta,
+        'sysmeta': sysmeta
     }
 
 
@@ -157,14 +175,12 @@ def headers_to_object_info(headers, status_int=HTTP_OK):
     """
     Construct a cacheable dict of object info based on response headers.
     """
-    headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
+    headers, meta, sysmeta = _prep_headers_to_info(headers, 'object')
     info = {'status': status_int,
             'length': headers.get('content-length'),
             'type': headers.get('content-type'),
             'etag': headers.get('etag'),
-            'meta': dict((key[14:], value)
-                         for key, value in headers.iteritems()
-                         if key.startswith('x-object-meta-'))
+            'meta': meta
             }
     return info
 
@@ -854,11 +870,10 @@ class Controller(object):
                            if k.lower().startswith(x_remove) or
                            k.lower() in self._x_remove_headers())
 
-        x_meta = 'x-%s-meta-' % st
         dst_headers.update((k.lower(), v)
                            for k, v in src_headers.iteritems()
                            if k.lower() in self.pass_through_headers or
-                           k.lower().startswith(x_meta))
+                           is_sys_or_user_meta(st, k))
 
     def generate_request_headers(self, orig_req=None, additional=None,
                                  transfer=False):

swift/proxy/controllers/obj.py

@@ -59,6 +59,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \
     HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \
     HTTPServerError, HTTPServiceUnavailable, Request, Response, \
     HTTPClientDisconnect, HTTPNotImplemented, HTTPException
+from swift.common.request_helpers import is_user_meta
 
 
 def segment_listing_iter(listing):
@@ -78,7 +79,7 @@ def copy_headers_into(from_r, to_r):
     """
     pass_headers = ['x-delete-at']
     for k, v in from_r.headers.items():
-        if k.lower().startswith('x-object-meta-') or k.lower() in pass_headers:
+        if is_user_meta('object', k) or k.lower() in pass_headers:
             to_r.headers[k] = v
 
 

swift/proxy/server.py

@@ -51,7 +51,17 @@ from swift.common.swob import HTTPBadRequest, HTTPForbidden, \
 # example, 'after: ["catch_errors", "bulk"]' would install this middleware
 # after catch_errors and bulk if both were present, but if bulk were absent,
 # would just install it after catch_errors.
-required_filters = [{'name': 'catch_errors'}]
+#
+# "after_fn" (optional) a function that takes a PipelineWrapper object as its
+# single argument and returns a list of middlewares that this middleware should
+# come after. This list overrides any defined by the "after" field.
+required_filters = [
+    {'name': 'catch_errors'},
+    {'name': 'gatekeeper',
+     'after_fn': lambda pipe: (['catch_errors']
+                               if pipe.startswith("catch_errors")
+                               else [])}
+]
 
 
 class Application(object):
@@ -505,7 +515,10 @@ class Application(object):
         for filter_spec in reversed(required_filters):
             filter_name = filter_spec['name']
             if filter_name not in pipe:
-                afters = filter_spec.get('after', [])
+                if 'after_fn' in filter_spec:
+                    afters = filter_spec['after_fn'](pipe)
+                else:
+                    afters = filter_spec.get('after', [])
                 insert_at = 0
                 for after in afters:
                     try:

test/unit/account/test_server.py

@@ -26,6 +26,7 @@ import xml.dom.minidom
 from swift.common.swob import Request
 from swift.account.server import AccountController, ACCOUNT_LISTING_LIMIT
 from swift.common.utils import normalize_timestamp, replication, public
+from swift.common.request_helpers import get_sys_meta_prefix
 
 
 class TestAccountController(unittest.TestCase):
@@ -371,6 +372,67 @@ class TestAccountController(unittest.TestCase):
         self.assertEqual(resp.status_int, 204)
         self.assert_('x-account-meta-test' not in resp.headers)
 
+    def test_PUT_GET_sys_metadata(self):
+        prefix = get_sys_meta_prefix('account')
+        hdr = '%stest' % prefix
+        hdr2 = '%stest2' % prefix
+        # Set metadata header
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
+            headers={'X-Timestamp': normalize_timestamp(1),
+                     hdr.title(): 'Value'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 201)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assertEqual(resp.headers.get(hdr), 'Value')
+        # Set another metadata header, ensuring old one doesn't disappear
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
+            headers={'X-Timestamp': normalize_timestamp(1),
+                     hdr2.title(): 'Value2'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assertEqual(resp.headers.get(hdr), 'Value')
+        self.assertEqual(resp.headers.get(hdr2), 'Value2')
+        # Update metadata header
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
+            headers={'X-Timestamp': normalize_timestamp(3),
+                     hdr.title(): 'New Value'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 202)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assertEqual(resp.headers.get(hdr), 'New Value')
+        # Send old update to metadata header
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
+            headers={'X-Timestamp': normalize_timestamp(2),
+                     hdr.title(): 'Old Value'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 202)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assertEqual(resp.headers.get(hdr), 'New Value')
+        # Remove metadata header (by setting it to empty)
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
+            headers={'X-Timestamp': normalize_timestamp(4),
+                     hdr.title(): ''})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 202)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assert_(hdr not in resp.headers)
+
     def test_PUT_invalid_partition(self):
         req = Request.blank('/sda1/./a', environ={'REQUEST_METHOD': 'PUT',
                                                   'HTTP_X_TIMESTAMP': '1'})
@@ -435,6 +497,59 @@ class TestAccountController(unittest.TestCase):
         self.assertEqual(resp.status_int, 204)
         self.assert_('x-account-meta-test' not in resp.headers)
 
+    def test_POST_HEAD_sys_metadata(self):
+        prefix = get_sys_meta_prefix('account')
+        hdr = '%stest' % prefix
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
+            headers={'X-Timestamp': normalize_timestamp(1)})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 201)
+        # Set metadata header
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
+            headers={'X-Timestamp': normalize_timestamp(1),
+                     hdr.title(): 'Value'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assertEqual(resp.headers.get(hdr), 'Value')
+        # Update metadata header
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
+            headers={'X-Timestamp': normalize_timestamp(3),
+                     hdr.title(): 'New Value'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assertEqual(resp.headers.get(hdr), 'New Value')
+        # Send old update to metadata header
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
+            headers={'X-Timestamp': normalize_timestamp(2),
+                     hdr.title(): 'Old Value'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assertEqual(resp.headers.get(hdr), 'New Value')
+        # Remove metadata header (by setting it to empty)
+        req = Request.blank(
+            '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
+            headers={'X-Timestamp': normalize_timestamp(4),
+                     hdr.title(): ''})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = req.get_response(self.controller)
+        self.assertEqual(resp.status_int, 204)
+        self.assert_(hdr not in resp.headers)
+
     def test_POST_invalid_partition(self):
         req = Request.blank('/sda1/./a', environ={'REQUEST_METHOD': 'POST',
                                                   'HTTP_X_TIMESTAMP': '1'})

test/unit/common/middleware/test_gatekeeper.py

@@ -0,0 +1,115 @@
+# Copyright (c) 2010-2012 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from swift.common.swob import Request, Response
+from swift.common.middleware import gatekeeper
+
+
+class FakeApp(object):
+    def __init__(self, headers={}):
+        self.headers = headers
+        self.req = None
+
+    def __call__(self, env, start_response):
+        self.req = Request(env)
+        return Response(request=self.req, body='FAKE APP',
+                        headers=self.headers)(env, start_response)
+
+
+class TestGatekeeper(unittest.TestCase):
+    methods = ['PUT', 'POST', 'GET', 'DELETE', 'HEAD', 'COPY', 'OPTIONS']
+
+    allowed_headers = {'xx-account-sysmeta-foo': 'value',
+                       'xx-container-sysmeta-foo': 'value',
+                       'xx-object-sysmeta-foo': 'value',
+                       'x-account-meta-foo': 'value',
+                       'x-container-meta-foo': 'value',
+                       'x-object-meta-foo': 'value',
+                       'x-timestamp-foo': 'value'}
+
+    sysmeta_headers = {'x-account-sysmeta-': 'value',
+                       'x-container-sysmeta-': 'value',
+                       'x-object-sysmeta-': 'value',
+                       'x-account-sysmeta-foo': 'value',
+                       'x-container-sysmeta-foo': 'value',
+                       'x-object-sysmeta-foo': 'value',
+                       'X-Account-Sysmeta-BAR': 'value',
+                       'X-Container-Sysmeta-BAR': 'value',
+                       'X-Object-Sysmeta-BAR': 'value'}
+
+    forbidden_headers_out = dict(sysmeta_headers)
+    forbidden_headers_in = dict(sysmeta_headers)
+
+    def _assertHeadersEqual(self, expected, actual):
+        for key in expected:
+            self.assertTrue(key.lower() in actual,
+                            '%s missing from %s' % (key, actual))
+
+    def _assertHeadersAbsent(self, unexpected, actual):
+        for key in unexpected:
+            self.assertTrue(key.lower() not in actual,
+                            '%s is in %s' % (key, actual))
+
+    def get_app(self, app, global_conf, **local_conf):
+        factory = gatekeeper.filter_factory(global_conf, **local_conf)
+        return factory(app)
+
+    def test_ok_header(self):
+        req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': 'PUT'},
+                            headers=self.allowed_headers)
+        fake_app = FakeApp()
+        app = self.get_app(fake_app, {})
+        resp = req.get_response(app)
+        self.assertEquals('200 OK', resp.status)
+        self.assertEquals(resp.body, 'FAKE APP')
+        self._assertHeadersEqual(self.allowed_headers, fake_app.req.headers)
+
+    def _test_reserved_header_removed_inbound(self, method):
+        headers = dict(self.forbidden_headers_in)
+        headers.update(self.allowed_headers)
+        req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': method},
+                            headers=headers)
+        fake_app = FakeApp()
+        app = self.get_app(fake_app, {})
+        resp = req.get_response(app)
+        self.assertEquals('200 OK', resp.status)
+        self._assertHeadersEqual(self.allowed_headers, fake_app.req.headers)
+        self._assertHeadersAbsent(self.forbidden_headers_in,
+                                  fake_app.req.headers)
+
+    def test_reserved_header_removed_inbound(self):
+        for method in self.methods:
+            self._test_reserved_header_removed_inbound(method)
+
+    def _test_reserved_header_removed_outbound(self, method):
+        headers = dict(self.forbidden_headers_out)
+        headers.update(self.allowed_headers)
+        req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': method})
+        fake_app = FakeApp(headers=headers)
+        app = self.get_app(fake_app, {})
+        resp = req.get_response(app)
+        self.assertEquals('200 OK', resp.status)
+        self._assertHeadersEqual(self.allowed_headers, resp.headers)
+        self._assertHeadersAbsent(self.forbidden_headers_out, resp.headers)
+
+    def test_reserved_header_removed_outbound(self):
+        for method in self.methods:
+            self._test_reserved_header_removed_outbound(method)
+
+
+if __name__ == '__main__':
+    unittest.main()

test/unit/common/test_request_helpers.py

@@ -0,0 +1,70 @@
+# Copyright (c) 2010-2012 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for swift.common.request_helpers"""
+
+import unittest
+from swift.common.request_helpers import is_sys_meta, is_user_meta, \
+    is_sys_or_user_meta, strip_sys_meta_prefix, strip_user_meta_prefix, \
+    remove_items
+
+server_types = ['account', 'container', 'object']
+
+
+class TestRequestHelpers(unittest.TestCase):
+    def test_is_user_meta(self):
+        m_type = 'meta'
+        for st in server_types:
+            self.assertTrue(is_user_meta(st, 'x-%s-%s-foo' % (st, m_type)))
+            self.assertFalse(is_user_meta(st, 'x-%s-%s-' % (st, m_type)))
+            self.assertFalse(is_user_meta(st, 'x-%s-%sfoo' % (st, m_type)))
+
+    def test_is_sys_meta(self):
+        m_type = 'sysmeta'
+        for st in server_types:
+            self.assertTrue(is_sys_meta(st, 'x-%s-%s-foo' % (st, m_type)))
+            self.assertFalse(is_sys_meta(st, 'x-%s-%s-' % (st, m_type)))
+            self.assertFalse(is_sys_meta(st, 'x-%s-%sfoo' % (st, m_type)))
+
+    def test_is_sys_or_user_meta(self):
+        m_types = ['sysmeta', 'meta']
+        for mt in m_types:
+            for st in server_types:
+                self.assertTrue(is_sys_or_user_meta(st, 'x-%s-%s-foo'
+                                                    % (st, mt)))
+                self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%s-'
+                                                     % (st, mt)))
+                self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%sfoo'
+                                                     % (st, mt)))
+
+    def test_strip_sys_meta_prefix(self):
+        mt = 'sysmeta'
+        for st in server_types:
+            self.assertEquals(strip_sys_meta_prefix(st, 'x-%s-%s-a'
+                                                    % (st, mt)), 'a')
+
+    def test_strip_user_meta_prefix(self):
+        mt = 'meta'
+        for st in server_types:
+            self.assertEquals(strip_user_meta_prefix(st, 'x-%s-%s-a'
+                                                     % (st, mt)), 'a')
+
+    def test_remove_items(self):
+        src = {'a': 'b',
+               'c': 'd'}
+        test = lambda x: x == 'a'
+        rem = remove_items(src, test)
+        self.assertEquals(src, {'c': 'd'})
+        self.assertEquals(rem, {'a': 'b'})

test/unit/common/test_wsgi.py

@@ -35,6 +35,7 @@ from eventlet import listen
 import mock
 
 import swift.common.middleware.catch_errors
+import swift.common.middleware.gatekeeper
 import swift.proxy.server
 
 from swift.common.swob import Request
@@ -143,6 +144,9 @@ class TestWSGI(unittest.TestCase):
         # verify pipeline is catch_errors -> proxy-server
         expected = swift.common.middleware.catch_errors.CatchErrorMiddleware
         self.assert_(isinstance(app, expected))
+        app = app.app
+        expected = swift.common.middleware.gatekeeper.GatekeeperMiddleware
+        self.assert_(isinstance(app, expected))
         self.assert_(isinstance(app.app, swift.proxy.server.Application))
         # config settings applied to app instance
         self.assertEquals(0.2, app.app.conn_timeout)
@@ -706,6 +710,31 @@ class TestPipelineWrapper(unittest.TestCase):
         # filters in the pipeline.
         return [c.entry_point_name for c in self.pipe.context.filter_contexts]
 
+    def test_startswith(self):
+        self.assertTrue(self.pipe.startswith("healthcheck"))
+        self.assertFalse(self.pipe.startswith("tempurl"))
+
+    def test_startswith_no_filters(self):
+        config = """
+        [DEFAULT]
+        swift_dir = TEMPDIR
+
+        [pipeline:main]
+        pipeline = proxy-server
+
+        [app:proxy-server]
+        use = egg:swift#proxy
+        conn_timeout = 0.2
+        """
+        contents = dedent(config)
+        with temptree(['proxy-server.conf']) as t:
+            conf_file = os.path.join(t, 'proxy-server.conf')
+            with open(conf_file, 'w') as f:
+                f.write(contents.replace('TEMPDIR', t))
+            ctx = wsgi.loadcontext(loadwsgi.APP, conf_file, global_conf={})
+            pipe = wsgi.PipelineWrapper(ctx)
+        self.assertTrue(pipe.startswith('proxy'))
+
     def test_insert_filter(self):
         original_modules = ['healthcheck', 'catch_errors', None]
         self.assertEqual(self._entry_point_names(), original_modules)
@@ -789,7 +818,7 @@ class TestPipelineModification(unittest.TestCase):
         swift_dir = TEMPDIR
 
         [pipeline:main]
-        pipeline = catch_errors proxy-server
+        pipeline = catch_errors gatekeeper proxy-server
 
         [app:proxy-server]
         use = egg:swift#proxy
@@ -797,6 +826,9 @@ class TestPipelineModification(unittest.TestCase):
 
         [filter:catch_errors]
         use = egg:swift#catch_errors
+
+        [filter:gatekeeper]
+        use = egg:swift#gatekeeper
         """
 
         contents = dedent(config)
@@ -809,6 +841,7 @@ class TestPipelineModification(unittest.TestCase):
 
         self.assertEqual(self.pipeline_modules(app),
                          ['swift.common.middleware.catch_errors',
+                         'swift.common.middleware.gatekeeper',
                           'swift.proxy.server'])
 
     def test_proxy_modify_wsgi_pipeline(self):
@@ -835,8 +868,11 @@ class TestPipelineModification(unittest.TestCase):
             _fake_rings(t)
             app = wsgi.loadapp(conf_file, global_conf={})
 
-        self.assertEqual(self.pipeline_modules(app)[0],
-                         'swift.common.middleware.catch_errors')
+        self.assertEqual(self.pipeline_modules(app),
+                         ['swift.common.middleware.catch_errors',
+                         'swift.common.middleware.gatekeeper',
+                         'swift.common.middleware.healthcheck',
+                         'swift.proxy.server'])
 
     def test_proxy_modify_wsgi_pipeline_ordering(self):
         config = """
@@ -892,6 +928,69 @@ class TestPipelineModification(unittest.TestCase):
             'swift.common.middleware.tempurl',
             'swift.proxy.server'])
 
+    def _proxy_modify_wsgi_pipeline(self, pipe):
+        config = """
+        [DEFAULT]
+        swift_dir = TEMPDIR
+
+        [pipeline:main]
+        pipeline = %s
+
+        [app:proxy-server]
+        use = egg:swift#proxy
+        conn_timeout = 0.2
+
+        [filter:healthcheck]
+        use = egg:swift#healthcheck
+
+        [filter:catch_errors]
+        use = egg:swift#catch_errors
+
+        [filter:gatekeeper]
+        use = egg:swift#gatekeeper
+        """
+        config = config % (pipe,)
+        contents = dedent(config)
+        with temptree(['proxy-server.conf']) as t:
+            conf_file = os.path.join(t, 'proxy-server.conf')
+            with open(conf_file, 'w') as f:
+                f.write(contents.replace('TEMPDIR', t))
+            _fake_rings(t)
+            app = wsgi.loadapp(conf_file, global_conf={})
+        return app
+
+    def test_gatekeeper_insertion_catch_errors_configured_at_start(self):
+        # catch_errors is configured at start, gatekeeper is not configured,
+        # so gatekeeper should be inserted just after catch_errors
+        pipe = 'catch_errors healthcheck proxy-server'
+        app = self._proxy_modify_wsgi_pipeline(pipe)
+        self.assertEqual(self.pipeline_modules(app), [
+            'swift.common.middleware.catch_errors',
+            'swift.common.middleware.gatekeeper',
+            'swift.common.middleware.healthcheck',
+            'swift.proxy.server'])
+
+    def test_gatekeeper_insertion_catch_errors_configured_not_at_start(self):
+        # catch_errors is configured, gatekeeper is not configured, so
+        # gatekeeper should be inserted at start of pipeline
+        pipe = 'healthcheck catch_errors proxy-server'
+        app = self._proxy_modify_wsgi_pipeline(pipe)
+        self.assertEqual(self.pipeline_modules(app), [
+            'swift.common.middleware.gatekeeper',
+            'swift.common.middleware.healthcheck',
+            'swift.common.middleware.catch_errors',
+            'swift.proxy.server'])
+
+    def test_catch_errors_gatekeeper_configured_not_at_start(self):
+        # catch_errors is configured, gatekeeper is configured, so
+        # no change should be made to pipeline
+        pipe = 'healthcheck catch_errors gatekeeper proxy-server'
+        app = self._proxy_modify_wsgi_pipeline(pipe)
+        self.assertEqual(self.pipeline_modules(app), [
+            'swift.common.middleware.healthcheck',
+            'swift.common.middleware.catch_errors',
+            'swift.common.middleware.gatekeeper',
+            'swift.proxy.server'])
 
 if __name__ == '__main__':
     unittest.main()

test/unit/container/test_server.py

@@ -31,6 +31,7 @@ import swift.container
 from swift.container import server as container_server
 from swift.common.utils import normalize_timestamp, mkdirs, public, replication
 from test.unit import fake_http_connect
+from swift.common.request_helpers import get_sys_meta_prefix
 
 
 @contextmanager
@@ -292,6 +293,64 @@ class TestContainerController(unittest.TestCase):
         self.assertEquals(resp.status_int, 204)
         self.assert_('x-container-meta-test' not in resp.headers)
 
+    def test_PUT_GET_sys_metadata(self):
+        prefix = get_sys_meta_prefix('container')
+        key = '%sTest' % prefix
+        key2 = '%sTest2' % prefix
+        # Set metadata header
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'X-Timestamp': normalize_timestamp(1),
+                                     key: 'Value'})
+        resp = self.controller.PUT(req)
+        self.assertEquals(resp.status_int, 201)
+        req = Request.blank('/sda1/p/a/c')
+        resp = self.controller.GET(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assertEquals(resp.headers.get(key.lower()), 'Value')
+        # Set another metadata header, ensuring old one doesn't disappear
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
+                            headers={'X-Timestamp': normalize_timestamp(1),
+                                     key2: 'Value2'})
+        resp = self.controller.POST(req)
+        self.assertEquals(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a/c')
+        resp = self.controller.GET(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assertEquals(resp.headers.get(key.lower()), 'Value')
+        self.assertEquals(resp.headers.get(key2.lower()), 'Value2')
+        # Update metadata header
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'X-Timestamp': normalize_timestamp(3),
+                                     key: 'New Value'})
+        resp = self.controller.PUT(req)
+        self.assertEquals(resp.status_int, 202)
+        req = Request.blank('/sda1/p/a/c')
+        resp = self.controller.GET(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assertEquals(resp.headers.get(key.lower()),
+                          'New Value')
+        # Send old update to metadata header
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'X-Timestamp': normalize_timestamp(2),
+                                     key: 'Old Value'})
+        resp = self.controller.PUT(req)
+        self.assertEquals(resp.status_int, 202)
+        req = Request.blank('/sda1/p/a/c')
+        resp = self.controller.GET(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assertEquals(resp.headers.get(key.lower()),
+                          'New Value')
+        # Remove metadata header (by setting it to empty)
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'X-Timestamp': normalize_timestamp(4),
+                                     key: ''})
+        resp = self.controller.PUT(req)
+        self.assertEquals(resp.status_int, 202)
+        req = Request.blank('/sda1/p/a/c')
+        resp = self.controller.GET(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assert_(key.lower() not in resp.headers)
+
     def test_PUT_invalid_partition(self):
         req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'PUT',
                                                     'HTTP_X_TIMESTAMP': '1'})
@@ -369,6 +428,56 @@ class TestContainerController(unittest.TestCase):
         self.assertEquals(resp.status_int, 204)
         self.assert_('x-container-meta-test' not in resp.headers)
 
+    def test_POST_HEAD_sys_metadata(self):
+        prefix = get_sys_meta_prefix('container')
+        key = '%sTest' % prefix
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
+                            headers={'X-Timestamp': normalize_timestamp(1)})
+        resp = self.controller.PUT(req)
+        self.assertEquals(resp.status_int, 201)
+        # Set metadata header
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
+                            headers={'X-Timestamp': normalize_timestamp(1),
+                                     key: 'Value'})
+        resp = self.controller.POST(req)
+        self.assertEquals(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = self.controller.HEAD(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assertEquals(resp.headers.get(key.lower()), 'Value')
+        # Update metadata header
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
+                            headers={'X-Timestamp': normalize_timestamp(3),
+                                     key: 'New Value'})
+        resp = self.controller.POST(req)
+        self.assertEquals(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = self.controller.HEAD(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assertEquals(resp.headers.get(key.lower()),
+                          'New Value')
+        # Send old update to metadata header
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
+                            headers={'X-Timestamp': normalize_timestamp(2),
+                                     key: 'Old Value'})
+        resp = self.controller.POST(req)
+        self.assertEquals(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = self.controller.HEAD(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assertEquals(resp.headers.get(key.lower()),
+                          'New Value')
+        # Remove metadata header (by setting it to empty)
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
+                            headers={'X-Timestamp': normalize_timestamp(4),
+                                     key: ''})
+        resp = self.controller.POST(req)
+        self.assertEquals(resp.status_int, 204)
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
+        resp = self.controller.HEAD(req)
+        self.assertEquals(resp.status_int, 204)
+        self.assert_(key.lower() not in resp.headers)
+
     def test_POST_invalid_partition(self):
         req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'POST',
                                                     'HTTP_X_TIMESTAMP': '1'})

test/unit/proxy/controllers/test_account.py

@@ -21,6 +21,7 @@ from swift.proxy import server as proxy_server
 from swift.proxy.controllers.base import headers_to_account_info
 from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH as MAX_ANAME_LEN
 from test.unit import fake_http_connect, FakeRing, FakeMemcache
+from swift.common.request_helpers import get_sys_meta_prefix
 
 
 class TestAccountController(unittest.TestCase):
@@ -95,6 +96,62 @@ class TestAccountController(unittest.TestCase):
             resp = controller.POST(req)
         self.assertEquals(400, resp.status_int)
 
+    def _make_callback_func(self, context):
+        def callback(ipaddr, port, device, partition, method, path,
+                     headers=None, query_string=None, ssl=False):
+            context['method'] = method
+            context['path'] = path
+            context['headers'] = headers or {}
+        return callback
+
+    def test_sys_meta_headers_PUT(self):
+        # check that headers in sys meta namespace make it through
+        # the proxy controller
+        sys_meta_key = '%stest' % get_sys_meta_prefix('account')
+        sys_meta_key = sys_meta_key.title()
+        user_meta_key = 'X-Account-Meta-Test'
+        # allow PUTs to account...
+        self.app.allow_account_management = True
+        controller = proxy_server.AccountController(self.app, 'a')
+        context = {}
+        callback = self._make_callback_func(context)
+        hdrs_in = {sys_meta_key: 'foo',
+                   user_meta_key: 'bar',
+                   'x-timestamp': '1.0'}
+        req = Request.blank('/v1/a', headers=hdrs_in)
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(200, 200, give_connect=callback)):
+            controller.PUT(req)
+        self.assertEqual(context['method'], 'PUT')
+        self.assertTrue(sys_meta_key in context['headers'])
+        self.assertEqual(context['headers'][sys_meta_key], 'foo')
+        self.assertTrue(user_meta_key in context['headers'])
+        self.assertEqual(context['headers'][user_meta_key], 'bar')
+        self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
+
+    def test_sys_meta_headers_POST(self):
+        # check that headers in sys meta namespace make it through
+        # the proxy controller
+        sys_meta_key = '%stest' % get_sys_meta_prefix('account')
+        sys_meta_key = sys_meta_key.title()
+        user_meta_key = 'X-Account-Meta-Test'
+        controller = proxy_server.AccountController(self.app, 'a')
+        context = {}
+        callback = self._make_callback_func(context)
+        hdrs_in = {sys_meta_key: 'foo',
+                   user_meta_key: 'bar',
+                   'x-timestamp': '1.0'}
+        req = Request.blank('/v1/a', headers=hdrs_in)
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(200, 200, give_connect=callback)):
+            controller.POST(req)
+        self.assertEqual(context['method'], 'POST')
+        self.assertTrue(sys_meta_key in context['headers'])
+        self.assertEqual(context['headers'][sys_meta_key], 'foo')
+        self.assertTrue(user_meta_key in context['headers'])
+        self.assertEqual(context['headers'][user_meta_key], 'bar')
+        self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
+
 
 if __name__ == '__main__':
     unittest.main()

test/unit/proxy/controllers/test_base.py

@@ -20,10 +20,11 @@ from swift.proxy.controllers.base import headers_to_container_info, \
     get_container_memcache_key, get_account_info, get_account_memcache_key, \
     get_object_env_key, _get_cache_key, get_info, get_object_info, \
     Controller, GetOrHeadHandler
-from swift.common.swob import Request, HTTPException
+from swift.common.swob import Request, HTTPException, HeaderKeyDict
 from swift.common.utils import split_path
 from test.unit import fake_http_connect, FakeRing, FakeMemcache
 from swift.proxy import server as proxy_server
+from swift.common.request_helpers import get_sys_meta_prefix
 
 
 FakeResponse_status_int = 201
@@ -365,6 +366,15 @@ class TestFuncs(unittest.TestCase):
         self.assertEquals(resp['meta']['whatevs'], 14)
         self.assertEquals(resp['meta']['somethingelse'], 0)
 
+    def test_headers_to_container_info_sys_meta(self):
+        prefix = get_sys_meta_prefix('container')
+        headers = {'%sWhatevs' % prefix: 14,
+                   '%ssomethingelse' % prefix: 0}
+        resp = headers_to_container_info(headers.items(), 200)
+        self.assertEquals(len(resp['sysmeta']), 2)
+        self.assertEquals(resp['sysmeta']['whatevs'], 14)
+        self.assertEquals(resp['sysmeta']['somethingelse'], 0)
+
     def test_headers_to_container_info_values(self):
         headers = {
             'x-container-read': 'readvalue',
@@ -396,6 +406,15 @@ class TestFuncs(unittest.TestCase):
         self.assertEquals(resp['meta']['whatevs'], 14)
         self.assertEquals(resp['meta']['somethingelse'], 0)
 
+    def test_headers_to_account_info_sys_meta(self):
+        prefix = get_sys_meta_prefix('account')
+        headers = {'%sWhatevs' % prefix: 14,
+                   '%ssomethingelse' % prefix: 0}
+        resp = headers_to_account_info(headers.items(), 200)
+        self.assertEquals(len(resp['sysmeta']), 2)
+        self.assertEquals(resp['sysmeta']['whatevs'], 14)
+        self.assertEquals(resp['sysmeta']['somethingelse'], 0)
+
     def test_headers_to_account_info_values(self):
         headers = {
             'x-account-object-count': '10',
@@ -473,3 +492,43 @@ class TestFuncs(unittest.TestCase):
                                    {'Range': 'bytes=-100'})
         handler.fast_forward(20)
         self.assertEquals(handler.backend_headers['Range'], 'bytes=-80')
+
+    def test_transfer_headers_with_sysmeta(self):
+        base = Controller(self.app)
+        good_hdrs = {'x-base-sysmeta-foo': 'ok',
+                     'X-Base-sysmeta-Bar': 'also ok'}
+        bad_hdrs = {'x-base-sysmeta-': 'too short'}
+        hdrs = dict(good_hdrs)
+        hdrs.update(bad_hdrs)
+        dst_hdrs = HeaderKeyDict()
+        base.transfer_headers(hdrs, dst_hdrs)
+        self.assertEqual(HeaderKeyDict(good_hdrs), dst_hdrs)
+
+    def test_generate_request_headers(self):
+        base = Controller(self.app)
+        src_headers = {'x-remove-base-meta-owner': 'x',
+                       'x-base-meta-size': '151M',
+                       'new-owner': 'Kun'}
+        req = Request.blank('/v1/a/c/o', headers=src_headers)
+        dst_headers = base.generate_request_headers(req, transfer=True)
+        expected_headers = {'x-base-meta-owner': '',
+                            'x-base-meta-size': '151M'}
+        for k, v in expected_headers.iteritems():
+            self.assertTrue(k in dst_headers)
+            self.assertEqual(v, dst_headers[k])
+        self.assertFalse('new-owner' in dst_headers)
+
+    def test_generate_request_headers_with_sysmeta(self):
+        base = Controller(self.app)
+        good_hdrs = {'x-base-sysmeta-foo': 'ok',
+                     'X-Base-sysmeta-Bar': 'also ok'}
+        bad_hdrs = {'x-base-sysmeta-': 'too short'}
+        hdrs = dict(good_hdrs)
+        hdrs.update(bad_hdrs)
+        req = Request.blank('/v1/a/c/o', headers=hdrs)
+        dst_headers = base.generate_request_headers(req, transfer=True)
+        for k, v in good_hdrs.iteritems():
+            self.assertTrue(k.lower() in dst_headers)
+            self.assertEqual(v, dst_headers[k.lower()])
+        for k, v in bad_hdrs.iteritems():
+            self.assertFalse(k.lower() in dst_headers)

test/unit/proxy/controllers/test_container.py

@@ -20,6 +20,7 @@ from swift.common.swob import Request
 from swift.proxy import server as proxy_server
 from swift.proxy.controllers.base import headers_to_container_info
 from test.unit import fake_http_connect, FakeRing, FakeMemcache
+from swift.common.request_helpers import get_sys_meta_prefix
 
 
 class TestContainerController(unittest.TestCase):
@@ -62,6 +63,61 @@ class TestContainerController(unittest.TestCase):
         for key in owner_headers:
             self.assertTrue(key in resp.headers)
 
+    def _make_callback_func(self, context):
+        def callback(ipaddr, port, device, partition, method, path,
+                     headers=None, query_string=None, ssl=False):
+            context['method'] = method
+            context['path'] = path
+            context['headers'] = headers or {}
+        return callback
+
+    def test_sys_meta_headers_PUT(self):
+        # check that headers in sys meta namespace make it through
+        # the container controller
+        sys_meta_key = '%stest' % get_sys_meta_prefix('container')
+        sys_meta_key = sys_meta_key.title()
+        user_meta_key = 'X-Container-Meta-Test'
+        controller = proxy_server.ContainerController(self.app, 'a', 'c')
+
+        context = {}
+        callback = self._make_callback_func(context)
+        hdrs_in = {sys_meta_key: 'foo',
+                   user_meta_key: 'bar',
+                   'x-timestamp': '1.0'}
+        req = Request.blank('/v1/a/c', headers=hdrs_in)
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(200, 200, give_connect=callback)):
+            controller.PUT(req)
+        self.assertEqual(context['method'], 'PUT')
+        self.assertTrue(sys_meta_key in context['headers'])
+        self.assertEqual(context['headers'][sys_meta_key], 'foo')
+        self.assertTrue(user_meta_key in context['headers'])
+        self.assertEqual(context['headers'][user_meta_key], 'bar')
+        self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
+
+    def test_sys_meta_headers_POST(self):
+        # check that headers in sys meta namespace make it through
+        # the container controller
+        sys_meta_key = '%stest' % get_sys_meta_prefix('container')
+        sys_meta_key = sys_meta_key.title()
+        user_meta_key = 'X-Container-Meta-Test'
+        controller = proxy_server.ContainerController(self.app, 'a', 'c')
+        context = {}
+        callback = self._make_callback_func(context)
+        hdrs_in = {sys_meta_key: 'foo',
+                   user_meta_key: 'bar',
+                   'x-timestamp': '1.0'}
+        req = Request.blank('/v1/a/c', headers=hdrs_in)
+        with mock.patch('swift.proxy.controllers.base.http_connect',
+                        fake_http_connect(200, 200, give_connect=callback)):
+            controller.POST(req)
+        self.assertEqual(context['method'], 'POST')
+        self.assertTrue(sys_meta_key in context['headers'])
+        self.assertEqual(context['headers'][sys_meta_key], 'foo')
+        self.assertTrue(user_meta_key in context['headers'])
+        self.assertEqual(context['headers'][user_meta_key], 'bar')
+        self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
+
 
 if __name__ == '__main__':
     unittest.main()

test/unit/proxy/test_server.py

@@ -340,7 +340,8 @@ class TestController(unittest.TestCase):
                               'container_count': '12345',
                               'total_object_count': None,
                               'bytes': None,
-                              'meta': {}}
+                              'meta': {},
+                              'sysmeta': {}}
             self.assertEquals(container_info,
                               self.memcache.get(cache_key))
 
@@ -366,7 +367,8 @@ class TestController(unittest.TestCase):
                             'container_count': None,  # internally keep None
                             'total_object_count': None,
                             'bytes': None,
-                            'meta': {}}
+                            'meta': {},
+                            'sysmeta': {}}
             self.assertEquals(account_info,
                               self.memcache.get(cache_key))