openstack/swift
Code Issues Proposed changes

TempURL and FormPost Middleware

Browse Source 
Change-Id: I8d2ce2abdfe3a44605c9441ad7b1abc6c77e282d
This commit is contained in:
gholt 2011-12-21 13:54:07 +00:00 committed by gholt
parent 338be6a681
commit 7fc1721d7d
11 changed files with 3359 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
bin/swift-form-signature Normal file
View File

@ -0,0 +1,70 @@
#!/usr/bin/env python
import hmac
from hashlib import sha1
from os.path import basename
from sys import argv, exit
from time import time
if __name__ == '__main__':
if len(argv) != 7:
prog = basename(argv[0])
print 'Syntax: %s <path> <redirect> <max_file_size> ' \
'<max_file_count> <seconds> <key>' % prog
print
print 'Where:'
print ' <path> The prefix to use for form uploaded'
print ' objects. For example:'
print ' /v1/account/container/object_prefix_ would'
print ' ensure all form uploads have that path'
print ' prepended to the browser-given file name.'
print ' <redirect> The URL to redirect the browser to after'
print ' the uploads have completed.'
print ' <max_file_size> The maximum file size per file uploaded.'
print ' <max_file_count> The maximum number of uploaded files'
print ' allowed.'
print ' <seconds> The number of seconds from now to allow'
print ' the form post to begin.'
print ' <key> The X-Account-Meta-Temp-URL-Key for the'
print ' account.'
print
print 'Example output:'
print ' Expires: 1323842228'
print ' Signature: 18de97e47345a82c4dbfb3b06a640dbb'
exit(1)
path, redirect, max_file_size, max_file_count, seconds, key = argv[1:]
try:
max_file_size = int(max_file_size)
except ValueError:
max_file_size = -1
if max_file_size < 0:
print 'Please use a <max_file_size> value greater than or equal to 0.'
exit(1)
try:
max_file_count = int(max_file_count)
except ValueError:
max_file_count = 0
if max_file_count < 1:
print 'Please use a positive <max_file_count> value.'
exit(1)
try:
expires = int(time() + int(seconds))
except ValueError:
expires = 0
if expires < 1:
print 'Please use a positive <seconds> value.'
exit(1)
parts = path.split('/', 4)
# Must be four parts, ['', 'v1', 'a', 'c'], must be a v1 request, have
# account and container values, and optionally have an object prefix.
if len(parts) < 4 or parts[0] or parts[1] != 'v1' or not parts[2] or \
not parts[3]:
print '<path> must point to a container at least.'
print 'For example: /v1/account/container'
print ' Or: /v1/account/container/object_prefix'
exit(1)
sig = hmac.new(key, '%s\n%s\n%s\n%s\n%s' % (path, redirect, max_file_size,
max_file_count, expires), sha1).hexdigest()
print ' Expires:', expires
print 'Signature:', sig

59
bin/swift-temp-url Normal file
View File

@ -0,0 +1,59 @@
#!/usr/bin/env python
import hmac
from hashlib import sha1
from os.path import basename
from sys import argv, exit
from time import time
if __name__ == '__main__':
if len(argv) != 5:
prog = basename(argv[0])
print 'Syntax: %s <method> <seconds> <path> <key>' % prog
print
print 'Where:'
print ' <method> The method to allow, GET or PUT.'
print ' Note: HEAD will also be allowed.'
print ' <seconds> The number of seconds from now to allow requests.'
print ' <path> The full path to the resource.'
print ' Example: /v1/AUTH_account/c/o'
print ' <key> The X-Account-Meta-Temp-URL-Key for the account.'
print
print 'Example output:'
print ' /v1/AUTH_account/c/o?temp_url_sig=34d49efc32fe6e3082e411e' \
'eeb85bd8a&temp_url_expires=1323482948'
print
print 'This can be used to form a URL to give out for the access '
print 'allowed. For example:'
print ' echo https://swift-cluster.example.com`%s GET 60 ' \
'/v1/AUTH_account/c/o mykey`' % prog
print
print 'Might output:'
print ' https://swift-cluster.example.com/v1/AUTH_account/c/o?' \
'temp_url_sig=34d49efc32fe6e3082e411eeeb85bd8a&' \
'temp_url_expires=1323482948'
exit(1)
method, seconds, path, key = argv[1:]
if method not in ('GET', 'PUT'):
print 'Please use either the GET or PUT method.'
exit(1)
try:
expires = int(time() + int(seconds))
except ValueError:
expires = 0
if expires < 1:
print 'Please use a positive <seconds> value.'
exit(1)
parts = path.split('/', 4)
# Must be five parts, ['', 'v1', 'a', 'c', 'o'], must be a v1 request, have
# account, container, and object values, and the object value can't just
# have '/'s.
if len(parts) != 5 or parts[0] or parts[1] != 'v1' or not parts[2] or \
not parts[3] or not parts[4].strip('/'):
print '<path> must point to an object.'
print 'For example: /v1/account/container/object'
exit(1)
sig = hmac.new(key, '%s\n%s\n%s' % (method, expires, path),
sha1).hexdigest()
print '%s?temp_url_sig=%s&temp_url_expires=%s' % (path, sig, expires)

14
doc/source/misc.rst
View File

@ -143,3 +143,17 @@ StaticWeb
.. automodule:: swift.common.middleware.staticweb
:members:
:show-inheritance:
TempURL
=======
.. automodule:: swift.common.middleware.tempurl
:members:
:show-inheritance:
FormPost
========
.. automodule:: swift.common.middleware.formpost
:members:
:show-inheritance:

35
etc/proxy-server.conf-sample
View File

@ -69,6 +69,11 @@ use = egg:swift#tempauth
# This is a comma separated list of hosts allowed to send X-Container-Sync-Key
# requests.
# allowed_sync_hosts = 127.0.0.1
# This allows middleware higher in the WSGI pipeline to override auth
# processing, useful for middleware such as tempurl and formpost. If you know
# you're not going to use such middleware and you want a bit of extra security,
# you can set this to false.
# allow_overrides = true
# Lastly, you need to list all the accounts/users you want here. The format is:
# user_<account>_<user> = <key> [group] [group] [...] [storage_url]
# There are special groups of:
@ -185,3 +190,33 @@ use = egg:swift#staticweb
# set access_log_facility = LOG_LOCAL0
# set access_log_level = INFO
# set log_headers = False
# Note: Put tempurl just before your auth filter(s) in the pipeline
[filter:tempurl]
use = egg:swift#tempurl
#
# The headers to remove from incoming requests. Simply a whitespace delimited
# list of header names and names can optionally end with '*' to indicate a
# prefix match. incoming_allow_headers is a list of exceptions to these
# removals.
# incoming_remove_headers = x-timestamp
#
# The headers allowed as exceptions to incoming_remove_headers. Simply a
# whitespace delimited list of header names and names can optionally end with
# '*' to indicate a prefix match.
# incoming_allow_headers =
#
# The headers to remove from outgoing responses. Simply a whitespace delimited
# list of header names and names can optionally end with '*' to indicate a
# prefix match. outgoing_allow_headers is a list of exceptions to these
# removals.
# outgoing_remove_headers = x-object-meta-*
#
# The headers allowed as exceptions to outgoing_remove_headers. Simply a
# whitespace delimited list of header names and names can optionally end with
# '*' to indicate a prefix match.
# outgoing_allow_headers = x-object-meta-public-*
# Note: Put formpost just before your auth filter(s) in the pipeline
[filter:formpost]
use = egg:swift#formpost

47
setup.py
View File

@ -41,25 +41,40 @@ setup(
],
install_requires=[], # removed for better compat
scripts=[
'bin/swift', 'bin/swift-account-auditor',
'bin/swift-account-audit', 'bin/swift-account-reaper',
'bin/swift-account-replicator', 'bin/swift-account-server',
'bin/swift',
'bin/swift-account-audit',
'bin/swift-account-auditor',
'bin/swift-account-reaper',
'bin/swift-account-replicator',
'bin/swift-account-server',
'bin/swift-bench',
'bin/swift-container-auditor',
'bin/swift-container-replicator', 'bin/swift-container-sync',
'bin/swift-container-server', 'bin/swift-container-updater',
'bin/swift-drive-audit', 'bin/swift-get-nodes',
'bin/swift-init', 'bin/swift-object-auditor',
'bin/swift-object-expirer', 'bin/swift-object-info',
'bin/swift-container-replicator',
'bin/swift-container-server',
'bin/swift-container-sync',
'bin/swift-container-updater',
'bin/swift-dispersion-populate',
'bin/swift-dispersion-report',
'bin/swift-drive-audit',
'bin/swift-form-signature',
'bin/swift-get-nodes',
'bin/swift-init',
'bin/swift-object-auditor',
'bin/swift-object-expirer',
'bin/swift-object-info',
'bin/swift-object-replicator',
'bin/swift-object-server',
'bin/swift-object-updater', 'bin/swift-proxy-server',
'bin/swift-ring-builder', 'bin/swift-stats-populate',
'bin/swift-object-updater',
'bin/swift-oldies',
'bin/swift-orphans',
'bin/swift-proxy-server',
'bin/swift-recon',
'bin/swift-recon-cron',
'bin/swift-ring-builder',
'bin/swift-stats-populate',
'bin/swift-stats-report',
'bin/swift-dispersion-populate', 'bin/swift-dispersion-report',
'bin/swift-bench',
'bin/swift-recon', 'bin/swift-recon-cron', 'bin/swift-orphans',
'bin/swift-oldies'
],
'bin/swift-temp-url',
],
entry_points={
'paste.app_factory': [
'proxy=swift.proxy.server:app_factory',
@ -78,6 +93,8 @@ setup(
'staticweb=swift.common.middleware.staticweb:filter_factory',
'tempauth=swift.common.middleware.tempauth:filter_factory',
'recon=swift.common.middleware.recon:filter_factory',
'tempurl=swift.common.middleware.tempurl:filter_factory',
'formpost=swift.common.middleware.formpost:filter_factory',
],
},
)

542
swift/common/middleware/formpost.py Normal file
View File

@ -0,0 +1,542 @@
# Copyright (c) 2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
FormPost Middleware
Translates a browser form post into a regular Swift object PUT.
The format of the form is::
<form action="<swift-url>" method="POST"
enctype="multipart/form-data">
<input type="hidden" name="redirect" value="<redirect-url>" />
<input type="hidden" name="max_file_size" value="<bytes>" />
<input type="hidden" name="max_file_count" value="<count>" />
<input type="hidden" name="expires" value="<unix-timestamp>" />
<input type="hidden" name="signature" value="<hmac>" />
<input type="file" name="file1" /><br />
<input type="submit" />
</form>
The <swift-url> is the URL to the Swift desination, such as::
https://swift-cluster.example.com/AUTH_account/container/object_prefix
The name of each file uploaded will be appended to the <swift-url>
given. So, you can upload directly to the root of container with a
url like::
https://swift-cluster.example.com/AUTH_account/container/
Optionally, you can include an object prefix to better separate
different users' uploads, such as::
https://swift-cluster.example.com/AUTH_account/container/object_prefix
Note the form method must be POST and the enctype must be set as
"multipart/form-data".
The redirect attribute is the URL to redirect the browser to after
the upload completes. The URL will have status and message query
parameters added to it, indicating the HTTP status code for the
upload (2xx is success) and a possible message for further
information if there was an error (such as "max_file_size exceeded").
The max_file_size attribute must be included and indicates the
largest single file upload that can be done, in bytes.
The max_file_count attribute must be included and indicates the
maximum number of files that can be uploaded with the form. Include
additional ``<input type="file" name="filexx" />`` attributes if
desired.
The expires attribute is the Unix timestamp before which the form
must be submitted before it's invalidated.
The signature attribute is the HMAC-SHA1 signature of the form. Here is
sample code for computing the signature::
import hmac
from hashlib import sha1
from time import time
path = '/v1/account/container/object_prefix'
redirect = 'https://myserver.com/some-page'
max_file_size = 104857600
max_file_count = 10
expires = int(time() + 600)
key = 'mykey'
hmac_body = '%s\\n%s\\n%s\\n%s\\n%s' % (path, redirect,
max_file_size, max_file_count, expires)
signature = hmac.new(key, hmac_body, sha1).hexdigest()
The key is the value of the X-Account-Meta-Temp-URL-Key header on the
account.
The command line tool ``swift-form-signature`` may be used (mostly
just when testing) to compute expires and signature.
Also note that the file attributes must be after the other attributes
in order to be processed correctly. If attributes come after the
file, they won't be sent with the subrequest (there is no way to
parse all the attributes on the server-side without reading the whole
thing into memory -- to service many requests, some with large files,
there just isn't enough memory on the server, so attributes following
the file are simply ignored).
"""
__all__ = ['FormPost', 'filter_factory', 'READ_CHUNK_SIZE', 'MAX_VALUE_LENGTH']
import hmac
import re
import rfc822
from hashlib import sha1
from StringIO import StringIO
from time import gmtime, strftime, time
from time import time
from urllib import quote, unquote
from swift.common.utils import get_logger
#: The size of data to read from the form at any given time.
READ_CHUNK_SIZE = 4096
#: The maximum size of any attribute's value. Any additional data will be
#: truncated.
MAX_VALUE_LENGTH = 4096
#: Regular expression to match form attributes.
ATTRIBUTES_RE = re.compile(r'(\w+)=(".*?"|[^";]+)(; ?|$)')
class FormInvalid(Exception):
pass
def _parse_attrs(header):
"""
Given the value of a header like:
Content-Disposition: form-data; name="somefile"; filename="test.html"
Return data like
("form-data", {"name": "somefile", "filename": "test.html"})
:param header: Value of a header (the part after the ': ').
:returns: (value name, dict) of the attribute data parsed (see above).
"""
attributes = {}
attrs = ''
if '; ' in header:
header, attrs = header.split('; ', 1)
m = True
while m:
m = ATTRIBUTES_RE.match(attrs)
if m:
attrs = attrs[len(m.group(0)):]
attributes[m.group(1)] = m.group(2).strip('"')
return header, attributes
class _IterRequestsFileLikeObject(object):
def __init__(self, wsgi_input, boundary, input_buffer):
self.no_more_data_for_this_file = False
self.no_more_files = False
self.wsgi_input = wsgi_input
self.boundary = boundary
self.input_buffer = input_buffer
def read(self, length=None):
if not length:
length = READ_CHUNK_SIZE
if self.no_more_data_for_this_file:
return ''
# read enough data to know whether we're going to run
# into a boundary in next [length] bytes
if len(self.input_buffer) < length + len(self.boundary) + 2:
to_read = length + len(self.boundary) + 2
while to_read > 0:
chunk = self.wsgi_input.read(to_read)
to_read -= len(chunk)
self.input_buffer += chunk
if not chunk:
self.no_more_files = True
break
boundary_pos = self.input_buffer.find(self.boundary)
# boundary does not exist in the next (length) bytes
if boundary_pos == -1 or boundary_pos > length:
ret = self.input_buffer[:length]
self.input_buffer = self.input_buffer[length:]
# if it does, just return data up to the boundary
else:
ret, self.input_buffer = self.input_buffer.split(self.boundary, 1)
self.no_more_files = self.input_buffer.startswith('--')
self.no_more_data_for_this_file = True
self.input_buffer = self.input_buffer[2:]
return ret
def readline(self):
if self.no_more_data_for_this_file:
return ''
boundary_pos = newline_pos = -1
while newline_pos < 0 and boundary_pos < 0:
chunk = self.wsgi_input.read(READ_CHUNK_SIZE)
self.input_buffer += chunk
newline_pos = self.input_buffer.find('\r\n')
boundary_pos = self.input_buffer.find(self.boundary)
if not chunk:
self.no_more_files = True
break
# found a newline
if newline_pos >= 0 and \
(boundary_pos < 0 or newline_pos < boundary_pos):
# Use self.read to ensure any logic there happens...
ret = ''
to_read = newline_pos + 2
while to_read > 0:
chunk = self.read(to_read)
# Should never happen since we're reading from input_buffer,
# but just for completeness...
if not chunk:
break
to_read -= len(chunk)
ret += chunk
return ret
else: # no newlines, just return up to next boundary
return self.read(len(self.input_buffer))
def _iter_requests(wsgi_input, boundary):
"""
Given a multi-part mime encoded input file object and boundary,
yield file-like objects for each part.
:param wsgi_input: The file-like object to read from.
:param boundary: The mime boundary to separate new file-like
objects on.
:returns: A generator of file-like objects for each part.
"""
boundary = '--' + boundary
if wsgi_input.readline().strip() != boundary:
raise FormInvalid('invalid starting boundary')
boundary = '\r\n' + boundary
input_buffer = ''
done = False
while not done:
it = _IterRequestsFileLikeObject(wsgi_input, boundary, input_buffer)
yield it
done = it.no_more_files
input_buffer = it.input_buffer
class _CappedFileLikeObject(object):
"""
A file-like object wrapping another file-like object that raises
an EOFError if the amount of data read exceeds a given
max_file_size.
:param fp: The file-like object to wrap.
:param max_file_size: The maximum bytes to read before raising an
EOFError.
"""
def __init__(self, fp, max_file_size):
self.fp = fp
self.max_file_size = max_file_size
self.amount_read = 0
def read(self, size=None):
ret = self.fp.read(size)
self.amount_read += len(ret)
if self.amount_read > self.max_file_size:
raise EOFError('max_file_size exceeded')
return ret
def readline(self):
ret = self.fp.readline()
self.amount_read += len(ret)
if self.amount_read > self.max_file_size:
raise EOFError('max_file_size exceeded')
return ret
class FormPost(object):
"""
FormPost Middleware
See above for a full description.
:param app: The next WSGI filter or app in the paste.deploy
chain.
:param conf: The configuration dict for the middleware.
"""
def __init__(self, app, conf):
#: The next WSGI application/filter in the paste.deploy pipeline.
self.app = app
#: The filter configuration dict.
self.conf = conf
#: The logger to use with this middleware.
self.logger = get_logger(conf, log_route='formpost')
def __call__(self, env, start_response):
"""
Main hook into the WSGI paste.deploy filter/app pipeline.
:param env: The WSGI environment dict.
:param start_response: The WSGI start_response hook.
:returns: Response as per WSGI.
"""
if env['REQUEST_METHOD'] == 'POST':
try:
content_type, attrs = \
_parse_attrs(env.get('CONTENT_TYPE') or '')
if content_type == 'multipart/form-data' and \
'boundary' in attrs:
resp_status = [0]
def _start_response(status, headers, exc_info=None):
resp_status[0] = int(status.split(' ', 1)[0])
start_response(status, headers, exc_info)
self._log_request(env, resp_status)
return self._translate_form(env, start_response,
attrs['boundary'])
except (FormInvalid, EOFError), err:
self._log_request(env, 400)
body = 'FormPost: %s' % err
start_response('400 Bad Request',
(('Content-Type', 'text/plain'),
('Content-Length', str(len(body)))))
return [body]
return self.app(env, start_response)
def _translate_form(self, env, start_response, boundary):
"""
Translates the form data into subrequests and issues a
response.
:param env: The WSGI environment dict.
:param start_response: The WSGI start_response hook.
:returns: Response as per WSGI.
"""
key = self._get_key(env)
status = message = ''
attributes = {}
file_count = 0
for fp in _iter_requests(env['wsgi.input'], boundary):
hdrs = rfc822.Message(fp, 0)
disp, attrs = \
_parse_attrs(hdrs.getheader('Content-Disposition', ''))
if disp == 'form-data' and attrs.get('filename'):
file_count += 1
try:
if file_count > int(attributes.get('max_file_count') or 0):
status = '400 Bad Request'
message = 'max file count exceeded'
break
except ValueError:
raise FormInvalid('max_file_count not an integer')
attributes['filename'] = attrs['filename'] or 'filename'
if 'content-type' not in attributes and 'content-type' in hdrs:
attributes['content-type'] = \
hdrs['Content-Type'] or 'application/octet-stream'
status, message = self._perform_subrequest(env, start_response,
attributes, fp, key)
if status[:1] != '2':
break
else:
data = ''
mxln = MAX_VALUE_LENGTH
while mxln:
chunk = fp.read(mxln)
if not chunk:
break
mxln -= len(chunk)
data += chunk
while fp.read(READ_CHUNK_SIZE):
pass
if 'name' in attrs:
attributes[attrs['name'].lower()] = data.rstrip('\r\n--')
if not status:
status = '400 Bad Request'
message = 'no files to process'
if not attributes.get('redirect'):
body = status
if message:
body = status + '\r\nFormPost: ' + message.title()
start_response(status, [('Content-Type', 'text/plain'),
('Content-Length', len(body))])
return [body]
status = status.split(' ', 1)[0]
body = '<html><body><p><a href="%s?status=%s&message=%s">Click to ' \
'continue...</a></p></body></html>' % \
(attributes['redirect'], quote(status), quote(message))
start_response('303 See Other',
[('Location', '%s?status=%s&message=%s' %
(attributes['redirect'], quote(status), quote(message))),
('Content-Length', str(len(body)))])
return [body]
def _perform_subrequest(self, env, start_response, attributes, fp, key):
"""
Performs the subrequest and returns a new response.
:param env: The WSGI environment dict.
:param start_response: The WSGI start_response hook.
:param attributes: dict of the attributes of the form so far.
:param fp: The file-like object containing the request body.
:param key: The account key to validate the signature with.
:returns: Response as per WSGI.
"""
if not key:
return '401 Unauthorized', 'invalid signature'
try:
max_file_size = int(attributes.get('max_file_size') or 0)
except ValueError:
raise FormInvalid('max_file_size not an integer')
subenv = {'REQUEST_METHOD': 'PUT',
'SCRIPT_NAME': '',
'SERVER_NAME': env['SERVER_NAME'],
'SERVER_PORT': env['SERVER_PORT'],
'SERVER_PROTOCOL': env['SERVER_PROTOCOL'],
'HTTP_TRANSFER_ENCODING': 'chunked',
'wsgi.input': _CappedFileLikeObject(fp, max_file_size),
'swift.cache': env['swift.cache']}
subenv['PATH_INFO'] = env['PATH_INFO']
if subenv['PATH_INFO'][-1] != '/' and \
subenv['PATH_INFO'].count('/') < 4:
subenv['PATH_INFO'] += '/'
subenv['PATH_INFO'] += attributes['filename'] or 'filename'
if 'content-type' in attributes:
subenv['CONTENT_TYPE'] = \
attributes['content-type'] or 'application/octet-stream'
try:
if int(attributes.get('expires') or 0) < time():
return '401 Unauthorized', 'form expired'
except ValueError:
raise FormInvalid('expired not an integer')
hmac_body = '%s\n%s\n%s\n%s\n%s' % (
env['PATH_INFO'],
attributes.get('redirect') or '',
attributes.get('max_file_size') or '0',
attributes.get('max_file_count') or '0',
attributes.get('expires') or '0'
)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
if sig != (attributes.get('signature') or 'invalid'):
return '401 Unauthorized', 'invalid signature'
subenv['swift.authorize'] = lambda req: None
subenv['swift.authorize_override'] = True
substatus = [None]
def _start_response(status, headers, exc_info=None):
substatus[0] = status
self.app(subenv, _start_response)
return substatus[0], ''
def _get_key(self, env):
"""
Returns the X-Account-Meta-Temp-URL-Key header value for the
account, or None if none is set.
:param env: The WSGI environment for the request.
:returns: X-Account-Meta-Temp-URL-Key str value, or None.
"""
parts = env['PATH_INFO'].split('/', 4)
if len(parts) < 4 or parts[0] or parts[1] != 'v1' or not parts[2] or \
not parts[3]:
return None
account = parts[2]
key = None
memcache = env.get('swift.cache')
if memcache:
key = memcache.get('temp-url-key/%s' % account)
if not key:
newenv = {'REQUEST_METHOD': 'HEAD', 'SCRIPT_NAME': '',
'PATH_INFO': '/v1/' + account, 'CONTENT_LENGTH': '0',
'SERVER_PROTOCOL': 'HTTP/1.0',
'HTTP_USER_AGENT': 'FormPost', 'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http', 'wsgi.input': StringIO('')}
for name in ('SERVER_NAME', 'SERVER_PORT', 'wsgi.errors',
'wsgi.multithread', 'wsgi.multiprocess',
'wsgi.run_once', 'swift.cache', 'swift.trans_id'):
if name in env:
newenv[name] = env[name]
newenv['swift.authorize'] = lambda req: None
newenv['swift.authorize_override'] = True
key = [None]
def _start_response(status, response_headers, exc_info=None):
for h, v in response_headers:
if h.lower() == 'x-account-meta-temp-url-key':
key[0] = v
self.app(newenv, _start_response)
key = key[0]
if key and memcache:
memcache.set('temp-url-key/%s' % account, key, timeout=60)
return key
def _log_request(self, env, response_status_int):
"""
Used when a request might not be logged by the underlying
WSGI application, but we'd still like to record what
happened. An early 401 Unauthorized is a good example of
this.
:param env: The WSGI environment for the request.
:param response_status_int: The HTTP status we'll be replying
to the request with.
"""
the_request = quote(unquote(env.get('PATH_INFO') or '/'))
if env.get('QUERY_STRING'):
the_request = the_request + '?' + env['QUERY_STRING']
client = env.get('HTTP_X_CLUSTER_CLIENT_IP')
if not client and 'HTTP_X_FORWARDED_FOR' in env:
# remote host for other lbs
client = env['HTTP_X_FORWARDED_FOR'].split(',')[0].strip()
if not client:
client = env.get('REMOTE_ADDR')
self.logger.info(' '.join(quote(str(x)) for x in (
client or '-',
env.get('REMOTE_ADDR') or '-',
strftime('%d/%b/%Y/%H/%M/%S', gmtime()),
env.get('REQUEST_METHOD') or 'GET',
the_request,
env.get('SERVER_PROTOCOL') or '1.0',
response_status_int,
env.get('HTTP_REFERER') or '-',
(env.get('HTTP_USER_AGENT') or '-') + ' FormPOST',
env.get('HTTP_X_AUTH_TOKEN') or '-',
'-',
'-',
'-',
env.get('swift.trans_id') or '-',
'-',
'-',
)))
def filter_factory(global_conf, **local_conf):
""" Returns the WSGI filter for use with paste.deploy. """
conf = global_conf.copy()
conf.update(local_conf)
return lambda app: FormPost(app, conf)

6
swift/common/middleware/tempauth.py
View File

@ -28,7 +28,7 @@ from webob.exc import HTTPBadRequest, HTTPForbidden, HTTPNotFound, \
from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed
from swift.common.utils import cache_from_env, get_logger, get_remote_client, \
split_path
split_path, TRUE_VALUES
class TempAuth(object):
@ -79,6 +79,8 @@ class TempAuth(object):
self.allowed_sync_hosts = [h.strip()
for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',')
if h.strip()]
self.allow_overrides = \
conf.get('allow_overrides', 't').lower() in TRUE_VALUES
self.users = {}
for conf_key in conf:
if conf_key.startswith('user_'):
@ -120,6 +122,8 @@ class TempAuth(object):
will be routed through the internal auth request handler (self.handle).
This is to handle granting tokens, etc.
"""
if self.allow_overrides and env.get('swift.authorize_override', False):
return self.app(env, start_response)
if env.get('PATH_INFO', '').startswith(self.auth_prefix):
return self.handle(env, start_response)
s3 = env.get('HTTP_AUTHORIZATION')

486
swift/common/middleware/tempurl.py Normal file
View File

@ -0,0 +1,486 @@
# Copyright (c) 2010-2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
TempURL Middleware
Allows the creation of URLs to provide temporary access to objects.
For example, a website may wish to provide a link to download a large
object in Swift, but the Swift account has no public access. The
website can generate a URL that will provide GET access for a limited
time to the resource. When the web browser user clicks on the link,
the browser will download the object directly from Swift, obviating
the need for the website to act as a proxy for the request.
If the user were to share the link with all his friends, or
accidentally post it on a forum, etc. the direct access would be
limited to the expiration time set when the website created the link.
To create such temporary URLs, first an X-Account-Meta-Temp-URL-Key
header must be set on the Swift account. Then, an HMAC-SHA1 (RFC 2104)
signature is generated using the HTTP method to allow (GET or PUT),
the Unix timestamp the access should be allowed until, the full path
to the object, and the key set on the account.
For example, here is code generating the signature for a GET for 60
seconds on /v1/AUTH_account/container/object::
import hmac
from hashlib import sha1
from time import time
method = 'GET'
expires = int(time() + 60)
path = '/v1/AUTH_account/container/object'
key = 'mykey'
hmac_body = '%s\\n%s\\n%s\\n%s' % (method, expires, path, key)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
Let's say the sig ends up equaling ee796f3a89cf82ef7a36ac75fed54b83
and expires ends up 1323479485. Then, for example, the website could
provide a link to::
https://swift-cluster.example.com/v1/AUTH_account/container/object?
temp_url_sig=ee796f3a89cf82ef7a36ac75fed54b83&temp_url_expires=1323479485
Any alteration of the resource path or query arguments would result
in 401 Unauthorized. Similary, a PUT where GET was the allowed method
would 401. HEAD is allowed if GET or PUT is allowed.
Using this in combination with browser form post translation
middleware could also allow direct-from-browser uploads to specific
locations in Swift.
Note that changing the X-Account-Meta-Temp-URL-Key will invalidate
any previously generated temporary URLs within 60 seconds (the
memcache time for the key).
"""
__all__ = ['TempURL', 'filter_factory',
'DEFAULT_INCOMING_REMOVE_HEADERS',
'DEFAULT_INCOMING_ALLOW_HEADERS',
'DEFAULT_OUTGOING_REMOVE_HEADERS',
'DEFAULT_OUTGOING_ALLOW_HEADERS']
import hmac
from hashlib import sha1
from os.path import basename
from StringIO import StringIO
from time import gmtime, strftime, time
from urllib import quote, unquote
from urlparse import parse_qs
from swift.common.utils import get_logger
#: Default headers to remove from incoming requests. Simply a whitespace
#: delimited list of header names and names can optionally end with '*' to
#: indicate a prefix match. DEFAULT_INCOMING_ALLOW_HEADERS is a list of
#: exceptions to these removals.
DEFAULT_INCOMING_REMOVE_HEADERS = 'x-timestamp'
#: Default headers as exceptions to DEFAULT_INCOMING_REMOVE_HEADERS. Simply a
#: whitespace delimited list of header names and names can optionally end with
#: '*' to indicate a prefix match.
DEFAULT_INCOMING_ALLOW_HEADERS = ''
#: Default headers to remove from outgoing responses. Simply a whitespace
#: delimited list of header names and names can optionally end with '*' to
#: indicate a prefix match. DEFAULT_OUTGOING_ALLOW_HEADERS is a list of
#: exceptions to these removals.
DEFAULT_OUTGOING_REMOVE_HEADERS = 'x-object-meta-*'
#: Default headers as exceptions to DEFAULT_OUTGOING_REMOVE_HEADERS. Simply a
#: whitespace delimited list of header names and names can optionally end with
#: '*' to indicate a prefix match.
DEFAULT_OUTGOING_ALLOW_HEADERS = 'x-object-meta-public-*'
class TempURL(object):
"""
WSGI Middleware to grant temporary URLs specific access to Swift
resources. See the overview for more information.
This middleware understands the following configuration settings::
incoming_remove_headers
The headers to remove from incoming requests. Simply a
whitespace delimited list of header names and names can
optionally end with '*' to indicate a prefix match.
incoming_allow_headers is a list of exceptions to these
removals.
Default: x-timestamp
incoming_allow_headers
The headers allowed as exceptions to
incoming_remove_headers. Simply a whitespace delimited
list of header names and names can optionally end with
'*' to indicate a prefix match.
Default: None
outgoing_remove_headers
The headers to remove from outgoing responses. Simply a
whitespace delimited list of header names and names can
optionally end with '*' to indicate a prefix match.
outgoing_allow_headers is a list of exceptions to these
removals.
Default: x-object-meta-*
outgoing_allow_headers
The headers allowed as exceptions to
outgoing_remove_headers. Simply a whitespace delimited
list of header names and names can optionally end with
'*' to indicate a prefix match.
Default: x-object-meta-public-*
:param app: The next WSGI filter or app in the paste.deploy
chain.
:param conf: The configuration dict for the middleware.
"""
def __init__(self, app, conf):
#: The next WSGI application/filter in the paste.deploy pipeline.
self.app = app
#: The filter configuration dict.
self.conf = conf
#: The logger to use with this middleware.
self.logger = get_logger(conf, log_route='tempurl')
headers = DEFAULT_INCOMING_REMOVE_HEADERS
if 'incoming_remove_headers' in conf:
headers = conf['incoming_remove_headers']
headers = \
['HTTP_' + h.upper().replace('-', '_') for h in headers.split()]
#: Headers to remove from incoming requests. Uppercase WSGI env style,
#: like `HTTP_X_PRIVATE`.
self.incoming_remove_headers = [h for h in headers if h[-1] != '*']
#: Header with match prefixes to remove from incoming requests.
#: Uppercase WSGI env style, like `HTTP_X_SENSITIVE_*`.
self.incoming_remove_headers_startswith = \
[h[:-1] for h in headers if h[-1] == '*']
headers = DEFAULT_INCOMING_ALLOW_HEADERS
if 'incoming_allow_headers' in conf:
headers = conf['incoming_allow_headers']
headers = \
['HTTP_' + h.upper().replace('-', '_') for h in headers.split()]
#: Headers to allow in incoming requests. Uppercase WSGI env style,
#: like `HTTP_X_MATCHES_REMOVE_PREFIX_BUT_OKAY`.
self.incoming_allow_headers = [h for h in headers if h[-1] != '*']
#: Header with match prefixes to allow in incoming requests. Uppercase
#: WSGI env style, like `HTTP_X_MATCHES_REMOVE_PREFIX_BUT_OKAY_*`.
self.incoming_allow_headers_startswith = \
[h[:-1] for h in headers if h[-1] == '*']
headers = DEFAULT_OUTGOING_REMOVE_HEADERS
if 'outgoing_remove_headers' in conf:
headers = conf['outgoing_remove_headers']
headers = [h.lower() for h in headers.split()]
#: Headers to remove from outgoing responses. Lowercase, like
#: `x-account-meta-temp-url-key`.
self.outgoing_remove_headers = [h for h in headers if h[-1] != '*']
#: Header with match prefixes to remove from outgoing responses.
#: Lowercase, like `x-account-meta-private-*`.
self.outgoing_remove_headers_startswith = \
[h[:-1] for h in headers if h[-1] == '*']
headers = DEFAULT_OUTGOING_ALLOW_HEADERS
if 'outgoing_allow_headers' in conf:
headers = conf['outgoing_allow_headers']
headers = [h.lower() for h in headers.split()]
#: Headers to allow in outgoing responses. Lowercase, like
#: `x-matches-remove-prefix-but-okay`.
self.outgoing_allow_headers = [h for h in headers if h[-1] != '*']
#: Header with match prefixes to allow in outgoing responses.
#: Lowercase, like `x-matches-remove-prefix-but-okay-*`.
self.outgoing_allow_headers_startswith = \
[h[:-1] for h in headers if h[-1] == '*']
def __call__(self, env, start_response):
"""
Main hook into the WSGI paste.deploy filter/app pipeline.
:param env: The WSGI environment dict.
:param start_response: The WSGI start_response hook.
:returns: Response as per WSGI.
"""
temp_url_sig, temp_url_expires = self._get_temp_url_info(env)
if temp_url_sig is None and temp_url_expires is None:
return self.app(env, start_response)
if not temp_url_sig or not temp_url_expires:
return self._invalid(env, start_response)
account = self._get_account(env)
if not account:
return self._invalid(env, start_response)
key = self._get_key(env, account)
if not key:
return self._invalid(env, start_response)
if env['REQUEST_METHOD'] == 'HEAD':
hmac_val = self._get_hmac(env, temp_url_expires, key,
request_method='GET')
if temp_url_sig != hmac_val:
hmac_val = self._get_hmac(env, temp_url_expires, key,
request_method='PUT')
if temp_url_sig != hmac_val:
return self._invalid(env, start_response)
else:
hmac_val = self._get_hmac(env, temp_url_expires, key)
if temp_url_sig != hmac_val:
return self._invalid(env, start_response)
self._clean_incoming_headers(env)
env['swift.authorize'] = lambda req: None
env['swift.authorize_override'] = True
def _start_response(status, headers, exc_info=None):
headers = self._clean_outgoing_headers(headers)
if env['REQUEST_METHOD'] == 'GET':
already = False
for h, v in headers:
if h.lower() == 'content-disposition':
already = True
break
if not already:
headers.append(('Content-Disposition',
'attachment; filename=%s' %
(quote(basename(env['PATH_INFO'])))))
return start_response(status, headers, exc_info)
return self.app(env, _start_response)
def _get_account(self, env):
"""
Returns just the account for the request, if it's an object GET, PUT,
or HEAD request; otherwise, None is returned.
:param env: The WSGI environment for the request.
:returns: Account str or None.
"""
account = None
if env['REQUEST_METHOD'] in ('GET', 'PUT', 'HEAD'):
parts = env['PATH_INFO'].split('/', 4)
# Must be five parts, ['', 'v1', 'a', 'c', 'o'], must be a v1
# request, have account, container, and object values, and the
# object value can't just have '/'s.
if len(parts) == 5 and not parts[0] and parts[1] == 'v1' and \
parts[2] and parts[3] and parts[4].strip('/'):
account = parts[2]
return account
def _get_temp_url_info(self, env):
"""
Returns the provided temporary URL parameters (sig, expires),
if given and syntactically valid. Either sig or expires could
be None if not provided. If provided, expires is also
converted to an int if possible or 0 if not, and checked for
expiration (returns 0 if expired).
:param env: The WSGI environment for the request.
:returns: (sig, expires) as described above.
"""
temp_url_sig = temp_url_expires = None
qs = parse_qs(env.get('QUERY_STRING', ''))
if 'temp_url_sig' in qs:
temp_url_sig = qs['temp_url_sig'][0]
if 'temp_url_expires' in qs:
try:
temp_url_expires = int(qs['temp_url_expires'][0])
except ValueError:
temp_url_expires = 0
if temp_url_expires < time():
temp_url_expires = 0
return temp_url_sig, temp_url_expires
def _get_key(self, env, account):
"""
Returns the X-Account-Meta-Temp-URL-Key header value for the
account, or None if none is set.
:param env: The WSGI environment for the request.
:param account: Account str.
:returns: X-Account-Meta-Temp-URL-Key str value, or None.
"""
key = None
memcache = env.get('swift.cache')
if memcache:
key = memcache.get('temp-url-key/%s' % account)
if not key:
newenv = {'REQUEST_METHOD': 'HEAD', 'SCRIPT_NAME': '',
'PATH_INFO': '/v1/' + account, 'CONTENT_LENGTH': '0',
'SERVER_PROTOCOL': 'HTTP/1.0',
'HTTP_USER_AGENT': 'TempURL', 'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http', 'wsgi.input': StringIO('')}
for name in ('SERVER_NAME', 'SERVER_PORT', 'wsgi.errors',
'wsgi.multithread', 'wsgi.multiprocess',
'wsgi.run_once', 'swift.cache', 'swift.trans_id'):
if name in env:
newenv[name] = env[name]
newenv['swift.authorize'] = lambda req: None
newenv['swift.authorize_override'] = True
key = [None]
def _start_response(status, response_headers, exc_info=None):
for h, v in response_headers:
if h.lower() == 'x-account-meta-temp-url-key':
key[0] = v
self.app(newenv, _start_response)
key = key[0]
if key and memcache:
memcache.set('temp-url-key/%s' % account, key, timeout=60)
return key
def _get_hmac(self, env, expires, key, request_method=None):
"""
Returns the hexdigest string of the HMAC-SHA1 (RFC 2104) for
the request.
:param env: The WSGI environment for the request.
:param expires: Unix timestamp as an int for when the URL
expires.
:param key: Key str, from the X-Account-Meta-Temp-URL-Key of
the account.
:param request_method: Optional override of the request in
the WSGI env. For example, if a HEAD
does not match, you may wish to
override with GET to still allow the
HEAD.
:returns: hexdigest str of the HMAC-SHA1 for the request.
"""
if not request_method:
request_method = env['REQUEST_METHOD']
return hmac.new(key, '%s\n%s\n%s' % (request_method, expires,
env['PATH_INFO']), sha1).hexdigest()
def _invalid(self, env, start_response):
"""
Performs the necessary steps to indicate a WSGI 401
Unauthorized response to the request.
:param env: The WSGI environment for the request.
:param start_response: The WSGI start_response hook.
:returns: 401 response as per WSGI.
"""
self._log_request(env, 401)
body = '401 Unauthorized: Temp URL invalid\n'
start_response('401 Unauthorized',
[('Content-Type', 'text/plain'),
('Content-Length', str(len(body)))])
if env['REQUEST_METHOD'] == 'HEAD':
return []
return [body]
def _clean_incoming_headers(self, env):
"""
Removes any headers from the WSGI environment as per the
middleware configuration for incoming requests.
:param env: The WSGI environment for the request.
"""
for h in env.keys():
remove = h in self.incoming_remove_headers
if not remove:
for p in self.incoming_remove_headers_startswith:
if h.startswith(p):
remove = True
break
if remove:
if h in self.incoming_allow_headers:
remove = False
if remove:
for p in self.incoming_allow_headers_startswith:
if h.startswith(p):
remove = False
break
if remove:
del env[h]
def _clean_outgoing_headers(self, headers):
"""
Removes any headers as per the middleware configuration for
outgoing responses.
:param headers: A WSGI start_response style list of headers,
[('header1', 'value), ('header2', 'value),
...]
:returns: The same headers list, but with some headers
removed as per the middlware configuration for
outgoing responses.
"""
headers = dict(headers)
for h in headers.keys():
remove = h in self.outgoing_remove_headers
if not remove:
for p in self.outgoing_remove_headers_startswith:
if h.startswith(p):
remove = True
break
if remove:
if h in self.outgoing_allow_headers:
remove = False
if remove:
for p in self.outgoing_allow_headers_startswith:
if h.startswith(p):
remove = False
break
if remove:
del headers[h]
return headers.items()
def _log_request(self, env, response_status_int):
"""
Used when a request might not be logged by the underlying
WSGI application, but we'd still like to record what
happened. An early 401 Unauthorized is a good example of
this.
:param env: The WSGI environment for the request.
:param response_status_int: The HTTP status we'll be replying
to the request with.
"""
the_request = quote(unquote(env.get('PATH_INFO') or '/'))
if env.get('QUERY_STRING'):
the_request = the_request + '?' + env['QUERY_STRING']
client = env.get('HTTP_X_CLUSTER_CLIENT_IP')
if not client and 'HTTP_X_FORWARDED_FOR' in env:
# remote host for other lbs
client = env['HTTP_X_FORWARDED_FOR'].split(',')[0].strip()
if not client:
client = env.get('REMOTE_ADDR')
self.logger.info(' '.join(quote(str(x)) for x in (
client or '-',
env.get('REMOTE_ADDR') or '-',
strftime('%d/%b/%Y/%H/%M/%S', gmtime()),
env.get('REQUEST_METHOD') or 'GET',
the_request,
env.get('SERVER_PROTOCOL') or '1.0',
response_status_int,
env.get('HTTP_REFERER') or '-',
(env.get('HTTP_USER_AGENT') or '-') + ' TempURL',
env.get('HTTP_X_AUTH_TOKEN') or '-',
'-',
'-',
'-',
env.get('swift.trans_id') or '-',
'-',
'-',
)))
def filter_factory(global_conf, **local_conf):
""" Returns the WSGI filter for use with paste.deploy. """
conf = global_conf.copy()
conf.update(local_conf)
return lambda app: TempURL(app, conf)

1443
test/unit/common/middleware/test_formpost.py Normal file
View File

File diff suppressed because it is too large Load Diff

26
test/unit/common/middleware/test_tempauth.py
View File

@ -150,6 +150,32 @@ class TestAuth(unittest.TestCase):
self.assertEquals(resp.environ['swift.authorize'],
self.test_auth.authorize)
def test_override_asked_for_but_not_allowed(self):
self.test_auth = \
auth.filter_factory({'allow_overrides': 'false'})(FakeApp())
req = self._make_request('/v1/AUTH_account',
environ={'swift.authorize_override': True})
resp = req.get_response(self.test_auth)
self.assertEquals(resp.status_int, 401)
self.assertEquals(resp.environ['swift.authorize'],
self.test_auth.authorize)
def test_override_asked_for_and_allowed(self):
self.test_auth = \
auth.filter_factory({'allow_overrides': 'true'})(FakeApp())
req = self._make_request('/v1/AUTH_account',
environ={'swift.authorize_override': True})
resp = req.get_response(self.test_auth)
self.assertEquals(resp.status_int, 404)
self.assertTrue('swift.authorize' not in resp.environ)
def test_override_default_allowed(self):
req = self._make_request('/v1/AUTH_account',
environ={'swift.authorize_override': True})
resp = req.get_response(self.test_auth)
self.assertEquals(resp.status_int, 404)
self.assertTrue('swift.authorize' not in resp.environ)
def test_auth_deny_non_reseller_prefix(self):
resp = self._make_request('/v1/BLAH_account',
headers={'X-Auth-Token': 'BLAH_t'}).get_response(self.test_auth)

647
test/unit/common/middleware/test_tempurl.py Normal file
View File

@ -0,0 +1,647 @@
# Copyright (c) 2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import hmac
import unittest
from hashlib import sha1
from contextlib import contextmanager
from time import time
from webob import Request, Response
from swift.common.middleware import tempauth, tempurl
class FakeMemcache(object):
def __init__(self):
self.store = {}
def get(self, key):
return self.store.get(key)
def set(self, key, value, timeout=0):
self.store[key] = value
return True
def incr(self, key, timeout=0):
self.store[key] = self.store.setdefault(key, 0) + 1
return self.store[key]
@contextmanager
def soft_lock(self, key, timeout=0, retries=5):
yield True
def delete(self, key):
try:
del self.store[key]
except Exception:
pass
return True
class FakeApp(object):
def __init__(self, status_headers_body_iter=None):
self.calls = 0
self.status_headers_body_iter = status_headers_body_iter
if not self.status_headers_body_iter:
self.status_headers_body_iter = iter([('404 Not Found', {
'x-test-header-one-a': 'value1',
'x-test-header-two-a': 'value2',
'x-test-header-two-b': 'value3'}, '')])
self.request = None
def __call__(self, env, start_response):
self.calls += 1
self.request = Request.blank('', environ=env)
if 'swift.authorize' in env:
resp = env['swift.authorize'](self.request)
if resp:
return resp(env, start_response)
status, headers, body = self.status_headers_body_iter.next()
return Response(status=status, headers=headers,
body=body)(env, start_response)
class TestTempURL(unittest.TestCase):
def setUp(self):
self.app = FakeApp()
self.auth = tempauth.filter_factory({})(self.app)
self.tempurl = tempurl.filter_factory({})(self.auth)
def _make_request(self, path, **kwargs):
req = Request.blank(path, **kwargs)
req.environ['swift.cache'] = FakeMemcache()
return req
def test_passthrough(self):
resp = self._make_request('/v1/a/c/o').get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' not in resp.body)
def test_get_valid(self):
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
self.assertEquals(resp.headers['content-disposition'],
'attachment; filename=o')
def test_put_not_allowed_by_get(self):
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'REQUEST_METHOD': 'PUT',
'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
def test_put_valid(self):
method = 'PUT'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'REQUEST_METHOD': 'PUT',
'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
def test_get_not_allowed_by_put(self):
method = 'PUT'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
def test_missing_sig(self):
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'QUERY_STRING': 'temp_url_expires=%s' % expires})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
def test_missing_expires(self):
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'QUERY_STRING': 'temp_url_sig=%s' % sig})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
def test_bad_path(self):
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
def test_no_key(self):
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
def test_head_allowed_by_get(self):
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'REQUEST_METHOD': 'HEAD',
'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
def test_head_allowed_by_put(self):
method = 'PUT'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'REQUEST_METHOD': 'HEAD',
'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
def test_head_otherwise_not_allowed(self):
method = 'PUT'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
# Deliberately fudge expires to show HEADs aren't just automatically
# allowed.
expires += 1
req = self._make_request(path,
environ={'REQUEST_METHOD': 'HEAD',
'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
def test_post_not_allowed(self):
method = 'POST'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'REQUEST_METHOD': 'POST',
'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
def test_delete_not_allowed(self):
method = 'DELETE'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'REQUEST_METHOD': 'DELETE',
'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
def test_unknown_not_allowed(self):
method = 'UNKNOWN'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'REQUEST_METHOD': 'UNKNOWN',
'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
def test_changed_path_invalid(self):
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path + '2',
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
def test_changed_sig_invalid(self):
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
if sig[-1] != '0':
sig = sig[:-1] + '0'
else:
sig = sig[:-1] + '1'
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
def test_changed_expires_invalid(self):
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' %
(sig, expires + 1)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
def test_different_key_invalid(self):
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key + '2')
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 401)
self.assertTrue('Temp URL invalid' in resp.body)
def test_removed_incoming_header(self):
self.tempurl = tempurl.filter_factory({
'incoming_remove_headers': 'x-remove-this'})(self.auth)
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path, headers={'x-remove-this': 'value'},
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
self.assertTrue('x-remove-this' not in self.app.request.headers)
def test_removed_incoming_headers_match(self):
self.tempurl = tempurl.filter_factory({
'incoming_remove_headers': 'x-remove-this-*',
'incoming_allow_headers': 'x-remove-this-except-this'})(self.auth)
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
headers={'x-remove-this-one': 'value1',
'x-remove-this-except-this': 'value2'},
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
self.assertTrue('x-remove-this-one' not in self.app.request.headers)
self.assertEquals(
self.app.request.headers['x-remove-this-except-this'], 'value2')
def test_removed_outgoing_header(self):
self.tempurl = tempurl.filter_factory({
'outgoing_remove_headers': 'x-test-header-one-a'})(self.auth)
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
self.assertTrue('x-test-header-one-a' not in resp.headers)
self.assertEquals(resp.headers['x-test-header-two-a'], 'value2')
def test_removed_outgoing_headers_match(self):
self.tempurl = tempurl.filter_factory({
'outgoing_remove_headers': 'x-test-header-two-*',
'outgoing_allow_headers': 'x-test-header-two-b'})(self.auth)
method = 'GET'
expires = int(time() + 86400)
path = '/v1/a/c/o'
key = 'abc'
hmac_body = '%s\n%s\n%s' % (method, expires, path)
sig = hmac.new(key, hmac_body, sha1).hexdigest()
req = self._make_request(path,
environ={'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)})
req.environ['swift.cache'].set('temp-url-key/a', key)
resp = req.get_response(self.tempurl)
self.assertEquals(resp.status_int, 404)
self.assertEquals(resp.headers['x-test-header-one-a'], 'value1')
self.assertTrue('x-test-header-two-a' not in resp.headers)
self.assertEquals(resp.headers['x-test-header-two-b'], 'value3')
def test_get_account(self):
self.assertEquals(self.tempurl._get_account({
'REQUEST_METHOD': 'HEAD', 'PATH_INFO': '/v1/a/c/o'}), 'a')
self.assertEquals(self.tempurl._get_account({
'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a/c/o'}), 'a')
self.assertEquals(self.tempurl._get_account({
'REQUEST_METHOD': 'PUT', 'PATH_INFO': '/v1/a/c/o'}), 'a')
self.assertEquals(self.tempurl._get_account({
'REQUEST_METHOD': 'POST', 'PATH_INFO': '/v1/a/c/o'}), None)
self.assertEquals(self.tempurl._get_account({
'REQUEST_METHOD': 'DELETE', 'PATH_INFO': '/v1/a/c/o'}), None)
self.assertEquals(self.tempurl._get_account({
'REQUEST_METHOD': 'UNKNOWN', 'PATH_INFO': '/v1/a/c/o'}), None)
self.assertEquals(self.tempurl._get_account({
'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a/c/'}), None)
self.assertEquals(self.tempurl._get_account({
'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a/c//////'}), None)
self.assertEquals(self.tempurl._get_account({
'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a/c///o///'}), 'a')
self.assertEquals(self.tempurl._get_account({
'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a/c'}), None)
self.assertEquals(self.tempurl._get_account({
'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a//o'}), None)
self.assertEquals(self.tempurl._get_account({
'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1//c/o'}), None)
self.assertEquals(self.tempurl._get_account({
'REQUEST_METHOD': 'GET', 'PATH_INFO': '//a/c/o'}), None)
self.assertEquals(self.tempurl._get_account({
'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v2/a/c/o'}), None)
def test_get_temp_url_info(self):
s = 'f5d5051bddf5df7e27c628818738334f'
e = int(time() + 86400)
self.assertEquals(self.tempurl._get_temp_url_info({'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (s, e)}), (s, e))
self.assertEquals(self.tempurl._get_temp_url_info({}), (None, None))
self.assertEquals(self.tempurl._get_temp_url_info({'QUERY_STRING':
'temp_url_expires=%s' % e}), (None, e))
self.assertEquals(self.tempurl._get_temp_url_info({'QUERY_STRING':
'temp_url_sig=%s' % s}), (s, None))
self.assertEquals(self.tempurl._get_temp_url_info({'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=bad' % s}), (s, 0))
e = int(time() - 1)
self.assertEquals(self.tempurl._get_temp_url_info({'QUERY_STRING':
'temp_url_sig=%s&temp_url_expires=%s' % (s, e)}), (s, 0))
def test_get_key_memcache(self):
self.app.status_headers_body_iter = iter([('404 Not Found', {}, '')])
self.assertEquals(
self.tempurl._get_key({}, 'a'), None)
self.app.status_headers_body_iter = iter([('404 Not Found', {}, '')])
self.assertEquals(
self.tempurl._get_key({'swift.cache': None}, 'a'), None)
mc = FakeMemcache()
self.app.status_headers_body_iter = iter([('404 Not Found', {}, '')])
self.assertEquals(
self.tempurl._get_key({'swift.cache': mc}, 'a'), None)
mc.set('temp-url-key/a', 'abc')
self.assertEquals(
self.tempurl._get_key({'swift.cache': mc}, 'a'), 'abc')
def test_get_key_from_source(self):
self.app.status_headers_body_iter = \
iter([('200 Ok', {'x-account-meta-temp-url-key': 'abc'}, '')])
mc = FakeMemcache()
self.assertEquals(
self.tempurl._get_key({'swift.cache': mc}, 'a'), 'abc')
self.assertEquals(mc.get('temp-url-key/a'), 'abc')
def test_get_hmac(self):
self.assertEquals(self.tempurl._get_hmac(
{'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a/c/o'},
1, 'abc'),
'026d7f7cc25256450423c7ad03fc9f5ffc1dab6d')
self.assertEquals(self.tempurl._get_hmac(
{'REQUEST_METHOD': 'HEAD', 'PATH_INFO': '/v1/a/c/o'},
1, 'abc', request_method='GET'),
'026d7f7cc25256450423c7ad03fc9f5ffc1dab6d')
def test_invalid(self):
def _start_response(status, headers, exc_info=None):
self.assertTrue(status, '401 Unauthorized')
self.assertTrue('Temp URL invalid' in
''.join(self.tempurl._invalid({'REQUEST_METHOD': 'GET'},
_start_response)))
self.assertEquals('',
''.join(self.tempurl._invalid({'REQUEST_METHOD': 'HEAD'},
_start_response)))
def test_clean_incoming_headers(self):
irh = ''
iah = ''
env = {'HTTP_TEST_HEADER': 'value'}
tempurl.TempURL(None, {'incoming_remove_headers': irh,
'incoming_allow_headers': iah})._clean_incoming_headers(env)
self.assertTrue('HTTP_TEST_HEADER' in env)
irh = 'test-header'
iah = ''
env = {'HTTP_TEST_HEADER': 'value'}
tempurl.TempURL(None, {'incoming_remove_headers': irh,
'incoming_allow_headers': iah})._clean_incoming_headers(env)
self.assertTrue('HTTP_TEST_HEADER' not in env)
irh = 'test-header-*'
iah = ''
env = {'HTTP_TEST_HEADER_ONE': 'value',
'HTTP_TEST_HEADER_TWO': 'value'}
tempurl.TempURL(None, {'incoming_remove_headers': irh,
'incoming_allow_headers': iah})._clean_incoming_headers(env)
self.assertTrue('HTTP_TEST_HEADER_ONE' not in env)
self.assertTrue('HTTP_TEST_HEADER_TWO' not in env)
irh = 'test-header-*'
iah = 'test-header-two'
env = {'HTTP_TEST_HEADER_ONE': 'value',
'HTTP_TEST_HEADER_TWO': 'value'}
tempurl.TempURL(None, {'incoming_remove_headers': irh,
'incoming_allow_headers': iah})._clean_incoming_headers(env)
self.assertTrue('HTTP_TEST_HEADER_ONE' not in env)
self.assertTrue('HTTP_TEST_HEADER_TWO' in env)
irh = 'test-header-* test-other-header'
iah = 'test-header-two test-header-yes-*'
env = {'HTTP_TEST_HEADER_ONE': 'value',
'HTTP_TEST_HEADER_TWO': 'value',
'HTTP_TEST_OTHER_HEADER': 'value',
'HTTP_TEST_HEADER_YES': 'value',
'HTTP_TEST_HEADER_YES_THIS': 'value'}
tempurl.TempURL(None, {'incoming_remove_headers': irh,
'incoming_allow_headers': iah})._clean_incoming_headers(env)
self.assertTrue('HTTP_TEST_HEADER_ONE' not in env)
self.assertTrue('HTTP_TEST_HEADER_TWO' in env)
self.assertTrue('HTTP_TEST_OTHER_HEADER' not in env)
self.assertTrue('HTTP_TEST_HEADER_YES' not in env)
self.assertTrue('HTTP_TEST_HEADER_YES_THIS' in env)
def test_clean_outgoing_headers(self):
orh = ''
oah = ''
hdrs = {'test-header': 'value'}
hdrs = dict(tempurl.TempURL(None,
{'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
)._clean_outgoing_headers(hdrs.iteritems()))
self.assertTrue('test-header' in hdrs)
orh = 'test-header'
oah = ''
hdrs = {'test-header': 'value'}
hdrs = dict(tempurl.TempURL(None,
{'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
)._clean_outgoing_headers(hdrs.iteritems()))
self.assertTrue('test-header' not in hdrs)
orh = 'test-header-*'
oah = ''
hdrs = {'test-header-one': 'value',
'test-header-two': 'value'}
hdrs = dict(tempurl.TempURL(None,
{'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
)._clean_outgoing_headers(hdrs.iteritems()))
self.assertTrue('test-header-one' not in hdrs)
self.assertTrue('test-header-two' not in hdrs)
orh = 'test-header-*'
oah = 'test-header-two'
hdrs = {'test-header-one': 'value',
'test-header-two': 'value'}
hdrs = dict(tempurl.TempURL(None,
{'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
)._clean_outgoing_headers(hdrs.iteritems()))
self.assertTrue('test-header-one' not in hdrs)
self.assertTrue('test-header-two' in hdrs)
orh = 'test-header-* test-other-header'
oah = 'test-header-two test-header-yes-*'
hdrs = {'test-header-one': 'value',
'test-header-two': 'value',
'test-other-header': 'value',
'test-header-yes': 'value',
'test-header-yes-this': 'value'}
hdrs = dict(tempurl.TempURL(None,
{'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
)._clean_outgoing_headers(hdrs.iteritems()))
self.assertTrue('test-header-one' not in hdrs)
self.assertTrue('test-header-two' in hdrs)
self.assertTrue('test-other-header' not in hdrs)
self.assertTrue('test-header-yes' not in hdrs)
self.assertTrue('test-header-yes-this' in hdrs)
if __name__ == '__main__':
unittest.main()