openstack/swift
Code Issues Proposed changes

Support listing endpoints for an object.

Browse Source 
Implements blueprint list-endpoints.

DocImpact: new middleware list_endpoints.

Change-Id: I0c4911ff726abd4cb8ce2b6245c99786ad46b410
This commit is contained in:
Sergey Lukjanov 2013-02-01 22:50:21 +04:00
parent 34beb92edb
commit 7d5095c122
5 changed files with 367 additions and 0 deletions
doc/source
misc.rst
etc
proxy-server.conf-sample
setup.py
swift/common/middleware
list_endpoints.py
test/unit/common/middleware
test_list_endpoints.py

8
doc/source/misc.rst

@ -202,3 +202,11 @@ Static Large Objects
.. automodule:: swift.common.middleware.slo
:members:
:show-inheritance:
List Endpoints
==============
.. automodule:: swift.common.middleware.list_endpoints
:members:
:show-inheritance:

4
etc/proxy-server.conf-sample

@ -316,6 +316,10 @@ use = egg:swift#name_check
# maximum_length = 255
# forbidden_regexp = /\./|/\.\./|/\.$|/\.\.$
[filter:list-endpoints]
use = egg:swift#list_endpoints
# list_endpoints_path = /endpoints/
[filter:proxy-logging]
use = egg:swift#proxy_logging
# If not set, logging directives from [DEFAULT] without "access_" will be used

2
setup.py

@ -107,6 +107,8 @@ setup(
'proxy_logging=swift.common.middleware.proxy_logging:'
'filter_factory',
'slo=swift.common.middleware.slo:filter_factory',
'list_endpoints=swift.common.middleware.list_endpoints:'
'filter_factory',
],
},
)

153
swift/common/middleware/list_endpoints.py Normal file

@ -0,0 +1,153 @@
# Copyright (c) 2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
List endpoints for an object, account or container.
This middleware makes it possible to integrate swift with software
that relies on data locality information to avoid network overhead,
such as Hadoop.
Answers requests of the form::
/endpoints/{account}/{container}/{object}
/endpoints/{account}/{container}
/endpoints/{account}
with a JSON-encoded list of endpoints of the form::
http://{server}:{port}/{dev}/{part}/{acc}/{cont}/{obj}
http://{server}:{port}/{dev}/{part}/{acc}/{cont}
http://{server}:{port}/{dev}/{part}/{acc}
correspondingly, e.g.::
http://10.1.1.1:6000/sda1/2/a/c2/o1
http://10.1.1.1:6000/sda1/2/a/c2
http://10.1.1.1:6000/sda1/2/a
The '/endpoints/' path is customizable ('list_endpoints_path'
configuration parameter).
Intended for consumption by third-party services living inside the
cluster (as the endpoints make sense only inside the cluster behind
the firewall); potentially written in a different language.
This is why it's provided as a REST API and not just a Python API:
to avoid requiring clients to write their own ring parsers in their
languages, and to avoid the necessity to distribute the ring file
to clients and keep it up-to-date.
Note that the call is not authenticated, which means that a proxy
with this middleware enabled should not be open to an untrusted
environment (everyone can query the locality data using this middleware).
"""
from urllib import quote, unquote
from swift.common.ring import Ring
from swift.common.utils import json, get_logger, split_path
from swift.common.swob import Request, Response
from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed
class ListEndpointsMiddleware(object):
"""
List endpoints for an object, account or container.
See above for a full description.
Uses configuration parameter `swift_dir` (default `/etc/swift`).
:param app: The next WSGI filter or app in the paste.deploy
chain.
:param conf: The configuration dict for the middleware.
"""
def __init__(self, app, conf):
self.app = app
self.logger = get_logger(conf, log_route='endpoints')
swift_dir = conf.get('swift_dir', '/etc/swift')
self.account_ring = Ring(swift_dir, ring_name='account')
self.container_ring = Ring(swift_dir, ring_name='container')
self.object_ring = Ring(swift_dir, ring_name='object')
self.endpoints_path = conf.get('list_endpoints_path', '/endpoints/')
if not self.endpoints_path.endswith('/'):
self.endpoints_path += '/'
def __call__(self, env, start_response):
request = Request(env)
if not request.path.startswith(self.endpoints_path):
return self.app(env, start_response)
if request.method != 'GET':
return HTTPMethodNotAllowed(
req=request, headers={"Allow": "GET"})(env, start_response)
try:
clean_path = request.path[len(self.endpoints_path) - 1:]
account, container, obj = \
split_path(clean_path, 1, 3, True)
except ValueError:
return HTTPBadRequest('No account specified')(env, start_response)
if account is not None:
account = unquote(account)
if container is not None:
container = unquote(container)
if obj is not None:
obj = unquote(obj)
if obj is not None:
partition, nodes = self.object_ring.get_nodes(
account, container, obj)
endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
'{account}/{container}/{obj}'
elif container is not None:
partition, nodes = self.container_ring.get_nodes(
account, container)
endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
'{account}/{container}'
else:
partition, nodes = self.account_ring.get_nodes(
account)
endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
'{account}'
endpoints = []
for node in nodes:
endpoint = endpoint_template.format(
ip=node['ip'],
port=node['port'],
device=node['device'],
partition=partition,
account=quote(account),
container=quote(container or ''),
obj=quote(obj or ''))
endpoints.append(endpoint)
return Response(json.dumps(endpoints),
content_type='application/json')(env, start_response)
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
def list_endpoints_filter(app):
return ListEndpointsMiddleware(app, conf)
return list_endpoints_filter

200
test/unit/common/middleware/test_list_endpoints.py Normal file

@ -0,0 +1,200 @@
# Copyright (c) 2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import array
import unittest
from shutil import rmtree
import os
from swift.common import ring, utils
from swift.common.utils import json
from swift.common.swob import Request, Response
from swift.common.middleware import list_endpoints
class FakeApp(object):
def __call__(self, env, start_response):
return Response(body="FakeApp")(env, start_response)
def start_response(*args):
pass
class TestListEndpoints(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
self.testdir = os.path.join(os.path.dirname(__file__), 'ring')
rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir)
accountgz = os.path.join(self.testdir, 'account.ring.gz')
containergz = os.path.join(self.testdir, 'container.ring.gz')
objectgz = os.path.join(self.testdir, 'object.ring.gz')
# Let's make the rings slightly different so we can test
# that the correct ring is consulted (e.g. we don't consult
# the object ring to get nodes for a container)
intended_replica2part2dev_id_a = [
array.array('H', [3, 1, 3, 1]),
array.array('H', [0, 3, 1, 4]),
array.array('H', [1, 4, 0, 3])]
intended_replica2part2dev_id_c = [
array.array('H', [4, 3, 0, 1]),
array.array('H', [0, 1, 3, 4]),
array.array('H', [3, 4, 0, 1])]
intended_replica2part2dev_id_o = [
array.array('H', [0, 1, 0, 1]),
array.array('H', [0, 1, 0, 1]),
array.array('H', [3, 4, 3, 4])]
intended_devs = [{'id': 0, 'zone': 0, 'weight': 1.0,
'ip': '10.1.1.1', 'port': 6000,
'device': 'sda1'},
{'id': 1, 'zone': 0, 'weight': 1.0,
'ip': '10.1.1.1', 'port': 6000,
'device': 'sdb1'},
None,
{'id': 3, 'zone': 2, 'weight': 1.0,
'ip': '10.1.2.1', 'port': 6000,
'device': 'sdc1'},
{'id': 4, 'zone': 2, 'weight': 1.0,
'ip': '10.1.2.2', 'port': 6000,
'device': 'sdd1'}]
intended_part_shift = 30
intended_reload_time = 15
ring.RingData(intended_replica2part2dev_id_a,
intended_devs, intended_part_shift).save(accountgz)
ring.RingData(intended_replica2part2dev_id_c,
intended_devs, intended_part_shift).save(containergz)
ring.RingData(intended_replica2part2dev_id_o,
intended_devs, intended_part_shift).save(objectgz)
self.app = FakeApp()
self.list_endpoints = list_endpoints.filter_factory(
{'swift_dir': self.testdir})(self.app)
def tearDown(self):
rmtree(self.testdir, ignore_errors=1)
def test_get_endpoint(self):
# Expected results for objects taken from test_ring
# Expected results for others computed by manually invoking
# ring.get_nodes().
resp = Request.blank('/endpoints/a/c/o1').get_response(
self.list_endpoints)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.content_type, 'application/json')
self.assertEquals(json.loads(resp.body), [
"http://10.1.1.1:6000/sdb1/1/a/c/o1",
"http://10.1.2.2:6000/sdd1/1/a/c/o1"
])
# Here, 'o1/' is the object name.
resp = Request.blank('/endpoints/a/c/o1/').get_response(
self.list_endpoints)
self.assertEquals(resp.status_int, 200)
self.assertEquals(json.loads(resp.body), [
"http://10.1.1.1:6000/sdb1/3/a/c/o1/",
"http://10.1.2.2:6000/sdd1/3/a/c/o1/"
])
resp = Request.blank('/endpoints/a/c2').get_response(
self.list_endpoints)
self.assertEquals(resp.status_int, 200)
self.assertEquals(json.loads(resp.body), [
"http://10.1.1.1:6000/sda1/2/a/c2",
"http://10.1.2.1:6000/sdc1/2/a/c2"
])
resp = Request.blank('/endpoints/a1').get_response(
self.list_endpoints)
self.assertEquals(resp.status_int, 200)
self.assertEquals(json.loads(resp.body), [
"http://10.1.2.1:6000/sdc1/0/a1",
"http://10.1.1.1:6000/sda1/0/a1",
"http://10.1.1.1:6000/sdb1/0/a1"
])
resp = Request.blank('/endpoints/').get_response(
self.list_endpoints)
self.assertEquals(resp.status_int, 400)
resp = Request.blank('/endpoints/a/c 2').get_response(
self.list_endpoints)
self.assertEquals(resp.status_int, 200)
self.assertEquals(json.loads(resp.body), [
"http://10.1.1.1:6000/sdb1/3/a/c%202",
"http://10.1.2.2:6000/sdd1/3/a/c%202"
])
resp = Request.blank('/endpoints/a/c%202').get_response(
self.list_endpoints)
self.assertEquals(resp.status_int, 200)
self.assertEquals(json.loads(resp.body), [
"http://10.1.1.1:6000/sdb1/3/a/c%202",
"http://10.1.2.2:6000/sdd1/3/a/c%202"
])
resp = Request.blank('/endpoints/ac%20count/con%20tainer/ob%20ject') \
.get_response(self.list_endpoints)
self.assertEquals(resp.status_int, 200)
self.assertEquals(json.loads(resp.body), [
"http://10.1.1.1:6000/sdb1/3/ac%20count/con%20tainer/ob%20ject",
"http://10.1.2.2:6000/sdd1/3/ac%20count/con%20tainer/ob%20ject"
])
resp = Request.blank('/endpoints/a/c/o1', {'REQUEST_METHOD': 'POST'}) \
.get_response(self.list_endpoints)
self.assertEquals(resp.status_int, 405)
self.assertEquals(resp.status, '405 Method Not Allowed')
self.assertEquals(resp.headers['allow'], 'GET')
resp = Request.blank('/not-endpoints').get_response(
self.list_endpoints)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.status, '200 OK')
self.assertEquals(resp.body, 'FakeApp')
# test custom path with trailing slash
custom_path_le = list_endpoints.filter_factory({
'swift_dir': self.testdir,
'list_endpoints_path': '/some/another/path/'
})(self.app)
resp = Request.blank('/some/another/path/a/c/o1') \
.get_response(custom_path_le)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.content_type, 'application/json')
self.assertEquals(json.loads(resp.body), [
"http://10.1.1.1:6000/sdb1/1/a/c/o1",
"http://10.1.2.2:6000/sdd1/1/a/c/o1"
])
# test ustom path without trailing slash
custom_path_le = list_endpoints.filter_factory({
'swift_dir': self.testdir,
'list_endpoints_path': '/some/another/path'
})(self.app)
resp = Request.blank('/some/another/path/a/c/o1') \
.get_response(custom_path_le)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.content_type, 'application/json')
self.assertEquals(json.loads(resp.body), [
"http://10.1.1.1:6000/sdb1/1/a/c/o1",
"http://10.1.2.2:6000/sdd1/1/a/c/o1"
])
if __name__ == '__main__':
unittest.main()