openstack/swift
Code Issues Proposed changes

Re-add cname lookup and domain remap middleware

Browse Source 
Revert "removed cname lookup middleware"

This reverts commit b47bcf19e4.

Revert "removed domain remap middleware"

This reverts commit 317cf868bdf66dbc17c02d4ca383efafa5e2f229.

Change-Id: I260498d555c93b28896ace48a6f0e96701cbcc38
This commit is contained in:
John Dickinson 2012-05-17 10:43:44 -05:00
parent 1c3b75c291
commit 1e90b61076
7 changed files with 615 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
doc/manpages/proxy-server.conf.5
View File

@ -255,6 +255,35 @@ to r. Will limit PUT, DELETE, and POST requests to /a/c/o. The default is ''.
.RE
.RS 0
.IP "\fB[filter:domain_remap]\fR"
.RE
Middleware that translates container and account parts of a domain to path parameters that the proxy server understands. The container.account.storageurl/object gets translated to container.account.storageurl/path_root/account/container/object and account.storageurl/path_root/container/object gets translated to account.storageurl/path_root/account/container/object
.RS 3
.IP \fBuse\fR
Entry point for paste.deploy for the domain_remap middleware. This is the reference to the installed python egg.
The default is \fBegg:swift#domain_remap\fR.
.IP "\fBset log_name\fR"
Label used when logging. The default is domain_remap.
.IP "\fBset log_headers\fR"
Enables the ability to log request headers. The default is False.
.IP \fBstorage_domain\fR
The domain to be used by the middleware.
.IP \fBpath_root\fR
The path root value for the storage URL. The default is v1.
.IP \fBreseller_prefixes\fR
Browsers can convert a host header to lowercase, so check that reseller
prefix on the account is the correct case. This is done by comparing the
items in the reseller_prefixes config option to the found prefix. If they
match except for case, the item from reseller_prefixes will be used
instead of the found reseller prefix. The reseller_prefixes list is exclusive.
If defined, any request with an account prefix not in that list will be ignored
by this middleware. Defaults to 'AUTH'.
.RE
.RS 0
.IP "\fB[filter:catch_errors]\fR"

11
etc/proxy-server.conf-sample
View File

@ -168,6 +168,17 @@ use = egg:swift#ratelimit
# container_ratelimit_10 = 50
# container_ratelimit_50 = 20
[filter:domain_remap]
use = egg:swift#domain_remap
# You can override the default log routing for this filter here:
# set log_name = domain_remap
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = False
# storage_domain = example.com
# path_root = v1
# reseller_prefixes = AUTH
[filter:catch_errors]
use = egg:swift#catch_errors
# You can override the default log routing for this filter here:

1
setup.py
View File

@ -86,6 +86,7 @@ setup(
'ratelimit=swift.common.middleware.ratelimit:filter_factory',
'cname_lookup=swift.common.middleware.cname_lookup:filter_factory',
'catch_errors=swift.common.middleware.catch_errors:filter_factory',
'domain_remap=swift.common.middleware.domain_remap:filter_factory',
'swift3=swift.common.middleware.swift3:filter_factory',
'staticweb=swift.common.middleware.staticweb:filter_factory',
'tempauth=swift.common.middleware.tempauth:filter_factory',

152
swift/common/middleware/cname_lookup.py Normal file
View File

@ -0,0 +1,152 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
CNAME Lookup Middleware
Middleware that translates an unknown domain in the host header to
something that ends with the configured storage_domain by looking up
the given domain's CNAME record in DNS.
This middleware will continue to follow a CNAME chain in DNS until it finds
a record ending in the configured storage domain or it reaches the configured
maximum lookup depth. If a match is found, the environment's Host header is
rewritten and the request is passed further down the WSGI chain.
"""
from webob import Request
from webob.exc import HTTPBadRequest
try:
import dns.resolver
from dns.exception import DNSException
from dns.resolver import NXDOMAIN, NoAnswer
except ImportError:
# catch this to allow docs to be built without the dependency
MODULE_DEPENDENCY_MET = False
else: # executed if the try block finishes with no errors
MODULE_DEPENDENCY_MET = True
from swift.common.utils import cache_from_env, get_logger
def lookup_cname(domain): # pragma: no cover
"""
Given a domain, returns its DNS CNAME mapping and DNS ttl.
:param domain: domain to query on
:returns: (ttl, result)
"""
try:
answer = dns.resolver.query(domain, 'CNAME').rrset
ttl = answer.ttl
result = answer.items[0].to_text()
result = result.rstrip('.')
return ttl, result
except (DNSException, NXDOMAIN, NoAnswer):
return 0, None
class CNAMELookupMiddleware(object):
"""
CNAME Lookup Middleware
See above for a full description.
:param app: The next WSGI filter or app in the paste.deploy
chain.
:param conf: The configuration dict for the middleware.
"""
def __init__(self, app, conf):
if not MODULE_DEPENDENCY_MET:
# reraise the exception if the dependency wasn't met
raise ImportError('dnspython is required for this module')
self.app = app
self.storage_domain = conf.get('storage_domain', 'example.com')
if self.storage_domain and self.storage_domain[0] != '.':
self.storage_domain = '.' + self.storage_domain
self.lookup_depth = int(conf.get('lookup_depth', '1'))
self.memcache = None
self.logger = get_logger(conf, log_route='cname-lookup')
def __call__(self, env, start_response):
if not self.storage_domain:
return self.app(env, start_response)
given_domain = env['HTTP_HOST']
port = ''
if ':' in given_domain:
given_domain, port = given_domain.rsplit(':', 1)
if given_domain == self.storage_domain[1:]: # strip initial '.'
return self.app(env, start_response)
a_domain = given_domain
if not a_domain.endswith(self.storage_domain):
if self.memcache is None:
self.memcache = cache_from_env(env)
error = True
for tries in xrange(self.lookup_depth):
found_domain = None
if self.memcache:
memcache_key = ''.join(['cname-', a_domain])
found_domain = self.memcache.get(memcache_key)
if not found_domain:
ttl, found_domain = lookup_cname(a_domain)
if self.memcache:
memcache_key = ''.join(['cname-', given_domain])
self.memcache.set(memcache_key, found_domain,
timeout=ttl)
if found_domain is None or found_domain == a_domain:
# no CNAME records or we're at the last lookup
error = True
found_domain = None
break
elif found_domain.endswith(self.storage_domain):
# Found it!
self.logger.info(
_('Mapped %(given_domain)s to %(found_domain)s') %
{'given_domain': given_domain,
'found_domain': found_domain})
if port:
env['HTTP_HOST'] = ':'.join([found_domain, port])
else:
env['HTTP_HOST'] = found_domain
error = False
break
else:
# try one more deep in the chain
self.logger.debug(_('Following CNAME chain for ' \
'%(given_domain)s to %(found_domain)s') %
{'given_domain': given_domain,
'found_domain': found_domain})
a_domain = found_domain
if error:
if found_domain:
msg = 'CNAME lookup failed after %d tries' % \
self.lookup_depth
else:
msg = 'CNAME lookup failed to resolve to a valid domain'
resp = HTTPBadRequest(request=Request(env), body=msg,
content_type='text/plain')
return resp(env, start_response)
return self.app(env, start_response)
def filter_factory(global_conf, **local_conf): # pragma: no cover
conf = global_conf.copy()
conf.update(local_conf)
def cname_filter(app):
return CNAMELookupMiddleware(app, conf)
return cname_filter

130
swift/common/middleware/domain_remap.py Normal file
View File

@ -0,0 +1,130 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Domain Remap Middleware
Middleware that translates container and account parts of a domain to
path parameters that the proxy server understands.
container.account.storageurl/object gets translated to
container.account.storageurl/path_root/account/container/object
account.storageurl/path_root/container/object gets translated to
account.storageurl/path_root/account/container/object
Browsers can convert a host header to lowercase, so check that reseller
prefix on the account is the correct case. This is done by comparing the
items in the reseller_prefixes config option to the found prefix. If they
match except for case, the item from reseller_prefixes will be used
instead of the found reseller prefix. The reseller_prefixes list is
exclusive. If defined, any request with an account prefix not in that list
will be ignored by this middleware. reseller_prefixes defaults to 'AUTH'.
Note that this middleware requires that container names and account names
(except as described above) must be DNS-compatible. This means that the
account name created in the system and the containers created by users
cannot exceed 63 characters or have UTF-8 characters. These are
restrictions over and above what swift requires and are not explicitly
checked. Simply put, the this middleware will do a best-effort attempt to
derive account and container names from elements in the domain name and
put those derived values into the URL path (leaving the Host header
unchanged).
Also note that using container sync with remapped domain names is not
advised. With container sync, you should use the true storage end points as
sync destinations.
"""
from webob import Request
from webob.exc import HTTPBadRequest
class DomainRemapMiddleware(object):
"""
Domain Remap Middleware
See above for a full description.
:param app: The next WSGI filter or app in the paste.deploy
chain.
:param conf: The configuration dict for the middleware.
"""
def __init__(self, app, conf):
self.app = app
self.storage_domain = conf.get('storage_domain', 'example.com')
if self.storage_domain and self.storage_domain[0] != '.':
self.storage_domain = '.' + self.storage_domain
self.path_root = conf.get('path_root', 'v1').strip('/')
prefixes = conf.get('reseller_prefixes', 'AUTH')
self.reseller_prefixes = [x.strip() for x in prefixes.split(',')
if x.strip()]
self.reseller_prefixes_lower = [x.lower()
for x in self.reseller_prefixes]
def __call__(self, env, start_response):
if not self.storage_domain:
return self.app(env, start_response)
given_domain = env['HTTP_HOST']
port = ''
if ':' in given_domain:
given_domain, port = given_domain.rsplit(':', 1)
if given_domain.endswith(self.storage_domain):
parts_to_parse = given_domain[:-len(self.storage_domain)]
parts_to_parse = parts_to_parse.strip('.').split('.')
len_parts_to_parse = len(parts_to_parse)
if len_parts_to_parse == 2:
container, account = parts_to_parse
elif len_parts_to_parse == 1:
container, account = None, parts_to_parse[0]
else:
resp = HTTPBadRequest(request=Request(env),
body='Bad domain in host header',
content_type='text/plain')
return resp(env, start_response)
if '_' not in account and '-' in account:
account = account.replace('-', '_', 1)
account_reseller_prefix = account.split('_', 1)[0].lower()
if account_reseller_prefix not in self.reseller_prefixes_lower:
# account prefix is not in config list. bail.
return self.app(env, start_response)
prefix_index = self.reseller_prefixes_lower.index(
account_reseller_prefix)
real_prefix = self.reseller_prefixes[prefix_index]
if not account.startswith(real_prefix):
account_suffix = account[len(real_prefix):]
account = real_prefix + account_suffix
path = env['PATH_INFO'].strip('/')
new_path_parts = ['', self.path_root, account]
if container:
new_path_parts.append(container)
if path.startswith(self.path_root):
path = path[len(self.path_root):].lstrip('/')
if path:
new_path_parts.append(path)
new_path = '/'.join(new_path_parts)
env['PATH_INFO'] = new_path
return self.app(env, start_response)
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
def domain_filter(app):
return DomainRemapMiddleware(app, conf)
return domain_filter

166
test/unit/common/middleware/test_cname_lookup.py Normal file
View File

@ -0,0 +1,166 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from nose import SkipTest
from webob import Request
try:
# this test requires the dnspython package to be installed
import dns.resolver
except ImportError:
skip = True
else: # executed if the try has no errors
skip = False
from swift.common.middleware import cname_lookup
class FakeApp(object):
def __call__(self, env, start_response):
return "FAKE APP"
def start_response(*args):
pass
class TestCNAMELookup(unittest.TestCase):
def setUp(self):
if skip:
raise SkipTest
self.app = cname_lookup.CNAMELookupMiddleware(FakeApp(),
{'lookup_depth': 2})
def test_passthrough(self):
def my_lookup(d):
return 0, d
cname_lookup.lookup_cname = my_lookup
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'foo.example.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, 'FAKE APP')
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'foo.example.com:8080'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, 'FAKE APP')
def test_good_lookup(self):
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'mysite.com'})
def my_lookup(d):
return 0, '%s.example.com' % d
cname_lookup.lookup_cname = my_lookup
resp = self.app(req.environ, start_response)
self.assertEquals(resp, 'FAKE APP')
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'mysite.com:8080'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, 'FAKE APP')
def test_lookup_chain_too_long(self):
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'mysite.com'})
def my_lookup(d):
if d == 'mysite.com':
site = 'level1.foo.com'
elif d == 'level1.foo.com':
site = 'level2.foo.com'
elif d == 'level2.foo.com':
site = 'bar.example.com'
return 0, site
cname_lookup.lookup_cname = my_lookup
resp = self.app(req.environ, start_response)
self.assertEquals(resp, ['CNAME lookup failed after 2 tries'])
def test_lookup_chain_bad_target(self):
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'mysite.com'})
def my_lookup(d):
return 0, 'some.invalid.site.com'
cname_lookup.lookup_cname = my_lookup
resp = self.app(req.environ, start_response)
self.assertEquals(resp,
['CNAME lookup failed to resolve to a valid domain'])
def test_something_weird(self):
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'mysite.com'})
def my_lookup(d):
return 0, None
cname_lookup.lookup_cname = my_lookup
resp = self.app(req.environ, start_response)
self.assertEquals(resp,
['CNAME lookup failed to resolve to a valid domain'])
def test_with_memcache(self):
def my_lookup(d):
return 0, '%s.example.com' % d
cname_lookup.lookup_cname = my_lookup
class memcache_stub(object):
def __init__(self):
self.cache = {}
def get(self, key):
return self.cache.get(key, None)
def set(self, key, value, *a, **kw):
self.cache[key] = value
memcache = memcache_stub()
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET',
'swift.cache': memcache},
headers={'Host': 'mysite.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, 'FAKE APP')
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET',
'swift.cache': memcache},
headers={'Host': 'mysite.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, 'FAKE APP')
def test_cname_matching_ending_not_domain(self):
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'foo.com'})
def my_lookup(d):
return 0, 'c.aexample.com'
cname_lookup.lookup_cname = my_lookup
resp = self.app(req.environ, start_response)
self.assertEquals(resp,
['CNAME lookup failed to resolve to a valid domain'])
def test_cname_configured_with_empty_storage_domain(self):
app = cname_lookup.CNAMELookupMiddleware(FakeApp(),
{'storage_domain': '',
'lookup_depth': 2})
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'c.a.example.com'})
def my_lookup(d):
return 0, None
cname_lookup.lookup_cname = my_lookup
resp = app(req.environ, start_response)
self.assertEquals(resp, 'FAKE APP')

126
test/unit/common/middleware/test_domain_remap.py Normal file
View File

@ -0,0 +1,126 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from webob import Request
from swift.common.middleware import domain_remap
class FakeApp(object):
def __call__(self, env, start_response):
return env['PATH_INFO']
def start_response(*args):
pass
class TestDomainRemap(unittest.TestCase):
def setUp(self):
self.app = domain_remap.DomainRemapMiddleware(FakeApp(), {})
def test_domain_remap_passthrough(self):
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'example.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, '/')
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'example.com:8080'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, '/')
def test_domain_remap_account(self):
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'AUTH_a.example.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, '/v1/AUTH_a')
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'AUTH-uuid.example.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, '/v1/AUTH_uuid')
def test_domain_remap_account_container(self):
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'c.AUTH_a.example.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, '/v1/AUTH_a/c')
def test_domain_remap_extra_subdomains(self):
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'x.y.c.AUTH_a.example.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, ['Bad domain in host header'])
def test_domain_remap_account_with_path_root(self):
req = Request.blank('/v1', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'AUTH_a.example.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, '/v1/AUTH_a')
def test_domain_remap_account_container_with_path_root(self):
req = Request.blank('/v1', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'c.AUTH_a.example.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, '/v1/AUTH_a/c')
def test_domain_remap_account_container_with_path(self):
req = Request.blank('/obj', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'c.AUTH_a.example.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, '/v1/AUTH_a/c/obj')
def test_domain_remap_account_container_with_path_root_and_path(self):
req = Request.blank('/v1/obj', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'c.AUTH_a.example.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, '/v1/AUTH_a/c/obj')
def test_domain_remap_account_matching_ending_not_domain(self):
req = Request.blank('/dontchange', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'c.aexample.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, '/dontchange')
def test_domain_remap_configured_with_empty_storage_domain(self):
self.app = domain_remap.DomainRemapMiddleware(FakeApp(),
{'storage_domain': ''})
req = Request.blank('/test', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'c.AUTH_a.example.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, '/test')
def test_domain_remap_configured_with_prefixes(self):
conf = {'reseller_prefixes': 'PREFIX'}
self.app = domain_remap.DomainRemapMiddleware(FakeApp(), conf)
req = Request.blank('/test', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'c.prefix_uuid.example.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, '/v1/PREFIX_uuid/c/test')
def test_domain_remap_configured_with_bad_prefixes(self):
conf = {'reseller_prefixes': 'UNKNOWN'}
self.app = domain_remap.DomainRemapMiddleware(FakeApp(), conf)
req = Request.blank('/test', environ={'REQUEST_METHOD': 'GET'},
headers={'Host': 'c.prefix_uuid.example.com'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, '/test')
if __name__ == '__main__':
unittest.main()