openstack/glance
Code Issues Proposed changes

Remove user and key from location in swift

Browse Source 
The image locations table stores the swift url for images
which includes the user and key values. This if exposed,
can cause security risk. Hence this patch, santizies
that information out of the location before storing
and plugs it back in when it is required.
Introduced a new configuration file that supports
multiple swift account references. It has the credentials and
authurl for each store.  It is specified using
'swift_store_config_file'.
In addition, this patch does the following things:

Differentiate user and system created swift locations

Currently we do not differentiate between user supplied
uri and system created locations that have the account
reference. This patch introduces new scheme:
'swift+config' for this purpose.

Image create in V1 should validate the uri in case where location isn't
specified.

This patch ensures that a store is not set while
creating an image or updating it.

Related to bp remove-sensitive-data-from-locations
Implements blueprint: support-multiple-swift-backends
Implements bp: v1-image-create-should-validate-the-location-uri
DocImpact

Co-authored by: sridevik <sridevi.koushik@rackspace.com>,
iccha-sethi <iccha.sethi@rackspace.com>,
amalabasha <amala.alungal@rackspace.com>

Change-Id: I75af34145521f533dcd6f5fd7690f5a68f3b44b3
This commit is contained in:
sridevik 2014-02-04 04:37:41 -06:00 committed by AmalaBasha
parent 95fc85d5bf
commit 63195aaa3b
14 changed files with 690 additions and 132 deletions

24
doc/source/configuring.rst

@ -636,13 +636,33 @@ proxy).
The number of times a Swift download will be retried before the request
fails.
Optional. Default: ``0``
Configuring Multiple Swift Accounts/Stores
------------------------------------------
In order to not store Swift account credentials in the database, and to
have support for multiple accounts (or multiple Swift backing stores), a
reference is stored in the database and the corresponding configuration
(credentials/ parameters) details are stored in the configuration file.
Optional. Default: not enabled.
The location for this file is specified using the ``swift_store_config_file`` config file
in the section ``[DEFAULT]``. **If an incorrect value is specified, Glance API Swift store
service will not be configured.**
* ``swift_store_config_file=PATH``
`This option is specific to the Swift storage backend.`
* ``default_swift_reference=DEFAULT_REFERENCE``
Required when multiple Swift accounts/backing stores are configured.
Can only be specified in configuration files.
`This option is specific to the Swift storage backend.`
Optional. Default: ``0``
It is the default swift reference that is used to add any new images.
* ``swift_store_auth_insecure``
If True, bypass SSL certificate verification for Swift.

9
etc/glance-api.conf

@ -327,6 +327,15 @@ swift_store_create_container_on_put = False
# the maximum object size in Swift, which is 5GB
swift_store_large_object_size = 5120
# swift_store_config_file = glance-swift.conf
# This file contains references for each of the configured
# Swift accounts/backing stores. If used, this option can prevent
# credentials being stored in the database. Using Swift references
# is disabled if this config is left blank.
# The reference to the default Swift parameters to use for adding new images.
# default_swift_reference = 'ref1'
# When doing a large object manifest, what size, in MB, should
# Glance write chunks to Swift? This amount of data is written
# to a temporary disk buffer during the process of chunking

21
etc/glance-swift.conf.sample Normal file

@ -0,0 +1,21 @@
# glance-swift.conf.sample
#
# This file is an example config file when
# multiple swift accounts/backing stores are enabled.
#
# Specify the reference name in []
# For each section, specify the auth_address, user and key.
#
# WARNING:
# * If any of auth_address, user or key is not specified,
# the glance-api's swift store will fail to configure
[ref1]
user = tenant:user1
key = key1
auth_address = auth123@example.com
[ref2]
user = user2
key = key2
auth_address = http://auth345@example.com

7
glance/api/v1/images.py

@ -51,6 +51,7 @@ from glance.store import get_known_stores
from glance.store import get_size_from_backend
from glance.store import get_store_from_location
from glance.store import get_store_from_scheme
from glance.store import validate_location
LOG = logging.getLogger(__name__)
SUPPORTED_PARAMS = glance.api.v1.SUPPORTED_PARAMS
@ -707,6 +708,12 @@ class Controller(controller.BaseController):
self.pool.spawn_n(self._upload_and_activate, req, image_meta)
else:
if location:
try:
validate_location(req.context, location)
except (exception.BadStoreUri) as bse:
raise HTTPBadRequest(explanation=unicode(bse),
request=req)
self._validate_image_for_activation(req, image_id, image_meta)
image_size_meta = image_meta.get('size')
if image_size_meta:

4
glance/common/exception.py

@ -142,6 +142,10 @@ class InvalidPropertyProtectionConfiguration(Invalid):
message = _("Invalid configuration in property protection file.")
class InvalidSwiftStoreConfiguration(Invalid):
message = _("Invalid configuration in glance-swift conf file.")
class InvalidFilterRangeValue(Invalid):
message = _("Unable to filter using the specified range.")

99
glance/common/swift_store_utils.py Normal file

@ -0,0 +1,99 @@
# Copyright 2014 Rackspace
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import ConfigParser
try:
from collections import OrderedDict
except ImportError:
from ordereddict import OrderedDict
from oslo.config import cfg
from glance.common import exception
from glance.openstack.common import log as logging
swift_opts = [
cfg.StrOpt('default_swift_reference',
default="ref1",
help=_('The reference to the default swift account/backing '
'store parameters to use for adding new images.')),
cfg.StrOpt('swift_store_auth_address',
help=_('The address where the Swift authentication service '
'is listening.(deprecated)')),
cfg.StrOpt('swift_store_user', secret=True,
help=_('The user to authenticate against the Swift '
'authentication service (deprecated)')),
cfg.StrOpt('swift_store_key', secret=True,
help=_('Auth key for the user authenticating against the '
'Swift authentication service. (deprecated)')),
cfg.StrOpt('swift_store_config_file', secret=True,
help=_('The config file that has the swift account(s)'
'configs.')),
]
# NOTE(bourke): The default dict_type is collections.OrderedDict in py27, but
# we must set manually for compatibility with py26
CONFIG = ConfigParser.SafeConfigParser(dict_type=OrderedDict)
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
CONF.register_opts(swift_opts)
def is_multiple_swift_store_accounts_enabled():
if CONF.swift_store_config_file is None:
return False
return True
class SwiftParams(object):
def __init__(self):
if is_multiple_swift_store_accounts_enabled():
self.params = self._load_config()
else:
self.params = self._form_default_params()
def _form_default_params(self):
default = {}
if CONF.swift_store_user and CONF.swift_store_key \
and CONF.swift_store_auth_address:
default['user'] = CONF.swift_store_user
default['key'] = CONF.swift_store_key
default['auth_address'] = CONF.swift_store_auth_address
return {CONF.default_swift_reference: default}
return {}
def _load_config(self):
try:
conf_file = CONF.find_file(CONF.swift_store_config_file)
CONFIG.read(conf_file)
except Exception as e:
msg = (_("swift config file %(conf_file)s:%(exc)s not found") %
{'conf_file': CONF.swift_store_config_file, 'exc': e})
LOG.error(msg)
raise exception.InvalidSwiftStoreConfiguration()
account_params = {}
account_references = CONFIG.sections()
for ref in account_references:
reference = {}
try:
reference['auth_address'] = CONFIG.get(ref, 'auth_address')
reference['user'] = CONFIG.get(ref, 'user')
reference['key'] = CONFIG.get(ref, 'key')
account_params[ref] = reference
except (ValueError, SyntaxError, ConfigParser.NoOptionError) as e:
LOG.exception(_("Invalid format of swift store config"
"cfg"))
return account_params

6
glance/store/__init__.py

@ -289,6 +289,12 @@ def get_size_from_backend(context, uri):
return store.get_size(loc)
def validate_location(context, uri):
loc = location.get_location_from_uri(uri)
store = get_store_from_uri(context, uri, loc)
store.validate_location(uri)
def delete_from_backend(context, uri, **kwargs):
"""Removes chunks of data from backend specified by uri"""
loc = location.get_location_from_uri(uri)

7
glance/store/base.py

@ -92,6 +92,13 @@ class Store(object):
"""
pass
def validate_location(self, location):
"""
Takes a location and validates it for the presence
of any account references
"""
pass
def get(self, location):
"""
Takes a `glance.store.location.Location` object that indicates

184
glance/store/swift.py

@ -23,9 +23,11 @@ import math
from oslo.config import cfg
import six.moves.urllib.parse as urlparse
import urllib
from glance.common import auth
from glance.common import exception
from glance.common import swift_store_utils
from glance.openstack.common import excutils
import glance.openstack.common.log as logging
import glance.store
@ -48,19 +50,10 @@ swift_opts = [
cfg.BoolOpt('swift_enable_snet', default=False,
help=_('Whether to use ServiceNET to communicate with the '
'Swift storage servers.')),
cfg.StrOpt('swift_store_auth_address',
help=_('The address where the Swift authentication service '
'is listening.')),
cfg.StrOpt('swift_store_user', secret=True,
help=_('The user to authenticate against the Swift '
'authentication service.')),
cfg.StrOpt('swift_store_key', secret=True,
help=_('Auth key for the user authenticating against the '
'Swift authentication service.')),
cfg.StrOpt('swift_store_auth_version', default='2',
help=_('Version of the authentication service to use. '
'Valid versions are 2 for keystone and 1 for swauth '
'and rackspace.')),
'and rackspace. (deprecated)')),
cfg.BoolOpt('swift_store_auth_insecure', default=False,
help=_('If True, swiftclient won\'t check for a valid SSL '
'certificate when authenticating.')),
@ -113,6 +106,8 @@ swift_opts = [
CONF = cfg.CONF
CONF.register_opts(swift_opts)
SWIFT_STORE_REF_PARAMS = swift_store_utils.SwiftParams().params
def swift_retry_iter(resp_iter, length, store, location):
length = length if length else (resp_iter.len
@ -179,11 +174,10 @@ class StoreLocation(glance.store.location.StoreLocation):
def _get_credstring(self):
if self.user and self.key:
return '%s:%s@' % (urlparse.quote(self.user),
urlparse.quote(self.key))
return '%s:%s' % (urllib.quote(self.user), urllib.quote(self.key))
return ''
def get_uri(self):
def get_uri(self, credentials_included=True):
auth_or_store_url = self.auth_or_store_url
if auth_or_store_url.startswith('http://'):
auth_or_store_url = auth_or_store_url[len('http://'):]
@ -195,36 +189,42 @@ class StoreLocation(glance.store.location.StoreLocation):
container = self.container.strip('/')
obj = self.obj.strip('/')
if not credentials_included:
#Used only in case of an add
#Get the current store from config
store = CONF.default_swift_reference
return '%s://%s/%s/%s' % ('swift+config', store, container, obj)
if self.scheme == 'swift+config':
if self.ssl_enabled == True:
self.scheme = 'swift+https'
else:
self.scheme = 'swift+http'
if credstring != '':
credstring = "%s@" % credstring
return '%s://%s%s/%s/%s' % (self.scheme, credstring, auth_or_store_url,
container, obj)
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python. It also deals with the peculiarity that new-style
Swift URIs have where a username can contain a ':', like so:
def _get_conf_value_from_account_ref(self, netloc):
try:
self.user = SWIFT_STORE_REF_PARAMS[netloc]['user']
self.key = SWIFT_STORE_REF_PARAMS[netloc]['key']
netloc = SWIFT_STORE_REF_PARAMS[netloc]['auth_address']
self.ssl_enabled = True
if netloc != '':
if netloc.startswith('http://'):
self.ssl_enabled = False
netloc = netloc[len('http://'):]
elif netloc.startswith('https://'):
netloc = netloc[len('https://'):]
except KeyError:
reason = _("Badly formed Swift URI. Credentials not found for"
"account reference")
LOG.debug(reason)
raise exception.BadStoreUri()
return netloc
swift://account:user:pass@authurl.com/container/obj
"""
# Make sure that URIs that contain multiple schemes, such as:
# swift://user:pass@http://authurl.com/v1/container/obj
# are immediately rejected.
if uri.count('://') != 1:
reason = ("URI cannot contain more than one occurrence "
"of a scheme. If you have specified a URI like "
"swift://user:pass@http://authurl.com/v1/container/obj"
", you need to change it to use the "
"swift+http:// scheme, like so: "
"swift+http://user:pass@authurl.com/v1/container/obj")
LOG.debug("Invalid store URI: %(reason)s", {'reason': reason})
raise exception.BadStoreUri(message=reason)
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('swift', 'swift+http', 'swift+https')
self.scheme = pieces.scheme
netloc = pieces.netloc
path = pieces.path.lstrip('/')
def _form_uri_parts(self, netloc, path):
if netloc != '':
# > Python 2.6.1
if '@' in netloc:
@ -242,16 +242,24 @@ class StoreLocation(glance.store.location.StoreLocation):
path = path[path.find('/'):].strip('/')
if creds:
cred_parts = creds.split(':')
if len(cred_parts) != 2:
if len(cred_parts) < 2:
reason = "Badly formed credentials in Swift URI."
LOG.debug(reason)
raise exception.BadStoreUri()
user, key = cred_parts
self.user = urlparse.unquote(user)
self.key = urlparse.unquote(key)
key = cred_parts.pop()
user = ':'.join(cred_parts)
creds = urllib.unquote(creds)
try:
self.user, self.key = creds.rsplit(':', 1)
except exception.BadStoreConfiguration:
self.user = urllib.unquote(user)
self.key = urllib.unquote(key)
else:
self.user = None
self.key = None
return netloc, path
def _form_auth_or_store_url(self, netloc, path):
path_parts = path.split('/')
try:
self.obj = path_parts.pop()
@ -265,11 +273,52 @@ class StoreLocation(glance.store.location.StoreLocation):
LOG.debug(reason)
raise exception.BadStoreUri()
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python. It also deals with the peculiarity that new-style
Swift URIs have where a username can contain a ':', like so:
swift://account:user:pass@authurl.com/container/obj
and for system created locations with account reference
swift+config://account_reference/container/obj
"""
# Make sure that URIs that contain multiple schemes, such as:
# swift://user:pass@http://authurl.com/v1/container/obj
# are immediately rejected.
if uri.count('://') != 1:
reason = _("URI cannot contain more than one occurrence "
"of a scheme. If you have specified a URI like "
"swift://user:pass@http://authurl.com/v1/container/obj"
", you need to change it to use the "
"swift+http:// scheme, like so: "
"swift+http://user:pass@authurl.com/v1/container/obj")
LOG.debug("Invalid store URI: %(reason)s", {'reason': reason})
raise exception.BadStoreUri(message=reason)
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('swift', 'swift+http', 'swift+https',
'swift+config')
self.scheme = pieces.scheme
netloc = pieces.netloc
path = pieces.path.lstrip('/')
# NOTE(Sridevi): Fix to map the account reference to the
# corresponding CONF value
if self.scheme == 'swift+config':
netloc = self._get_conf_value_from_account_ref(netloc)
else:
netloc, path = self._form_uri_parts(netloc, path)
self._form_auth_or_store_url(netloc, path)
@property
def swift_url(self):
"""
Creates a fully-qualified auth url that the Swift client library can
use. The scheme for the auth_url is determined using the scheme
Creates a fully-qualified auth address that the Swift client library
can use. The scheme for the auth_address is determined using the scheme
included in the `location` field.
HTTPS is assumed, unless 'swift+http' is specified.
@ -277,6 +326,11 @@ class StoreLocation(glance.store.location.StoreLocation):
if self.auth_or_store_url.startswith('http'):
return self.auth_or_store_url
else:
if self.scheme == 'swift+config':
if self.ssl_enabled == True:
self.scheme = 'swift+https'
else:
self.scheme = 'swift+http'
if self.scheme in ('swift+https', 'swift'):
auth_scheme = 'https://'
else:
@ -296,7 +350,7 @@ class BaseStore(glance.store.base.Store):
CHUNKSIZE = 65536
def get_schemes(self):
return ('swift+https', 'swift', 'swift+http')
return ('swift+https', 'swift', 'swift+http', 'swift+config')
def configure(self):
_obj_size = self._option_get('swift_store_large_object_size')
@ -363,8 +417,8 @@ class BaseStore(glance.store.base.Store):
def _option_get(self, param):
result = getattr(CONF, param)
if not result:
reason = (_("Could not find %(param)s in configuration "
"options.") % {'param': param})
reason = (_("Could not find %(param)s in configuration options.")
% param)
LOG.error(reason)
raise exception.BadStoreConfiguration(store_name="swift",
reason=reason)
@ -491,8 +545,13 @@ class BaseStore(glance.store.base.Store):
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
if swift_store_utils.is_multiple_swift_store_accounts_enabled():
include_creds = False
else:
include_creds = True
return (location.get_uri(), image_size, obj_etag, {})
return (location.get_uri(credentials_included=include_creds),
image_size, obj_etag, {})
except swiftclient.ClientException as e:
if e.http_status == httplib.CONFLICT:
raise exception.Duplicate(_("Swift already has an image at "
@ -587,14 +646,29 @@ class SingleTenantStore(BaseStore):
self.auth_version = self._option_get('swift_store_auth_version')
def configure_add(self):
self.auth_address = self._option_get('swift_store_auth_address')
default_swift_reference = \
SWIFT_STORE_REF_PARAMS.get(
CONF.default_swift_reference)
if default_swift_reference:
self.auth_address = default_swift_reference.get('auth_address')
if (not default_swift_reference) or (not self.auth_address):
reason = _("A value for swift_store_auth_address is required.")
LOG.error(reason)
raise exception.BadStoreConfiguration(store_name="swift",
reason=reason)
if self.auth_address.startswith('http://'):
self.scheme = 'swift+http'
else:
self.scheme = 'swift+https'
self.container = CONF.swift_store_container
self.user = self._option_get('swift_store_user')
self.key = self._option_get('swift_store_key')
self.user = default_swift_reference.get('user')
self.key = default_swift_reference.get('key')
if not (self.user or self.key):
reason = _("A value for swift_store_ref_params is required.")
LOG.error(reason)
raise exception.BadStoreConfiguration(store_name="swift",
reason=reason)
def create_location(self, image_id):
specs = {'scheme': self.scheme,
@ -605,6 +679,12 @@ class SingleTenantStore(BaseStore):
'key': self.key}
return StoreLocation(specs)
def validate_location(self, uri):
pieces = urlparse.urlparse(uri)
if pieces.scheme in ['swift+config']:
reason = (_("Location credentials are invalid"))
raise exception.BadStoreUri(message=reason)
def get_connection(self, location):
if not location.user:
reason = "Location is missing user:password information."

34
glance/tests/etc/glance-swift.conf Normal file

@ -0,0 +1,34 @@
[ref1]
user = tenant:user1
key = key1
auth_address = example.com
[ref2]
user = user2
key = key2
auth_address = http://example.com
[store_2]
user = tenant:user1
key = key1
auth_address= https://localhost:8080
[store_3]
user= tenant:user2
key= key2
auth_address= https://localhost:8080
[store_4]
user = tenant:user1
key = key1
auth_address = http://localhost:80
[store_5]
user = tenant:user1
key = key1
auth_address = http://localhost
[store_6]
user = tenant:user1
key = key1
auth_address = https://localhost/v1

89
glance/tests/unit/common/test_swift_store_utils.py Normal file

@ -0,0 +1,89 @@
# Copyright 2014 Rackspace
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import fixtures
from glance.common import exception
from glance.common import swift_store_utils
from glance.tests.unit import base
class TestSwiftParams(base.IsolatedUnitTest):
def setUp(self):
conf_file = "glance-swift.conf"
test_dir = self.useFixture(fixtures.TempDir()).path
self.swift_config_file = self._copy_data_file(conf_file, test_dir)
self.config(swift_store_config_file=self.swift_config_file)
super(TestSwiftParams, self).setUp()
def tearDown(self):
super(TestSwiftParams, self).tearDown()
def test_multiple_swift_account_enabled(self):
self.config(swift_store_config_file="glance-swift.conf")
self.assertTrue(
swift_store_utils.is_multiple_swift_store_accounts_enabled())
def test_multiple_swift_account_disabled(self):
self.config(swift_store_config_file=None)
self.assertFalse(
swift_store_utils.is_multiple_swift_store_accounts_enabled())
def test_swift_config_file_doesnt_exist(self):
self.config(swift_store_config_file='fake-file.conf')
self.assertRaises(exception.InvalidSwiftStoreConfiguration,
swift_store_utils.SwiftParams)
def test_swift_config_uses_default_values_multiple_account_disabled(self):
default_user = 'user_default'
default_key = 'key_default'
default_auth_address = 'auth@default.com'
default_account_reference = 'ref_default'
confs = {'swift_store_config_file': None,
'swift_store_user': default_user,
'swift_store_key': default_key,
'swift_store_auth_address': default_auth_address,
'default_swift_reference': default_account_reference}
self.config(**confs)
swift_params = swift_store_utils.SwiftParams().params
self.assertEqual(1, len(swift_params.keys()))
self.assertEqual(default_user,
swift_params[default_account_reference]['user']
)
self.assertEqual(default_key,
swift_params[default_account_reference]['key']
)
self.assertEqual(default_auth_address,
swift_params[default_account_reference]
['auth_address']
)
def test_swift_store_config_validates_for_creds_auth_address(self):
swift_params = swift_store_utils.SwiftParams().params
self.assertEqual('tenant:user1',
swift_params['ref1']['user']
)
self.assertEqual('key1',
swift_params['ref1']['key']
)
self.assertEqual('example.com',
swift_params['ref1']['auth_address'])
self.assertEqual('user2',
swift_params['ref2']['user'])
self.assertEqual('key2',
swift_params['ref2']['key'])
self.assertEqual('http://example.com',
swift_params['ref2']['auth_address']
)

50
glance/tests/unit/test_store_location.py

@ -12,6 +12,8 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import fixtures
import mock
from glance.common import exception
@ -27,6 +29,15 @@ import glance.store.vmware_datastore
from glance.tests.unit import base
CONF = {'default_store': 'file',
'swift_store_auth_address': 'localhost:8080',
'swift_store_container': 'glance',
'swift_store_user': 'user',
'swift_store_key': 'key',
'default_swift_reference': 'store_1'
}
class TestStoreLocation(base.StoreClearingUnitTest):
def setUp(self):
@ -45,6 +56,9 @@ class TestStoreLocation(base.StoreClearingUnitTest):
"glance.store.gridfs.Store",
"glance.store.vmware_datastore.Store",
])
conf = CONF.copy()
self.config(**conf)
reload(glance.store.swift)
super(TestStoreLocation, self).setUp()
def test_get_location_from_uri_back_to_uri(self):
@ -160,6 +174,42 @@ class TestStoreLocation(base.StoreClearingUnitTest):
"""
Test the specific StoreLocation for the Swift store
"""
uri = 'swift+config://store_1/images/1'
loc = glance.store.swift.StoreLocation({})
loc.parse_uri(uri)
self.assertEqual("swift+config", loc.scheme)
self.assertEqual("localhost:8080", loc.auth_or_store_url)
self.assertEqual("https://localhost:8080", loc.swift_url)
self.assertEqual("images", loc.container)
self.assertEqual("1", loc.obj)
self.assertEqual('user', loc.user)
self.assertEqual('swift+https://user:key@localhost:8080/images/1',
loc.get_uri())
conf_file = "glance-swift.conf"
test_dir = self.useFixture(fixtures.TempDir()).path
self.swift_config_file = self._copy_data_file(conf_file, test_dir)
conf = CONF.copy()
conf.update({'swift_store_config_file': self.swift_config_file})
self.config(**conf)
reload(glance.store.swift)
uri = 'swift+config://store_2/images/1'
loc = glance.store.swift.StoreLocation({})
loc.parse_uri(uri)
self.assertEqual("swift+config", loc.scheme)
self.assertEqual("localhost:8080", loc.auth_or_store_url)
self.assertEqual("https://localhost:8080", loc.swift_url)
self.assertEqual("images", loc.container)
self.assertEqual("1", loc.obj)
self.assertEqual('tenant:user1', loc.user)
self.assertEqual('key1', loc.key)
self.assertEqual('swift+https://tenant%3Auser1:key1@localhost:8080'
'/images/1',
loc.get_uri())
uri = 'swift://example.com/images/1'
loc = glance.store.swift.StoreLocation({})
loc.parse_uri(uri)

265
glance/tests/unit/test_swift_store.py

@ -15,6 +15,8 @@
"""Tests the Swift backend store"""
import copy
import fixtures
import hashlib
import httplib
import mock
@ -23,20 +25,22 @@ import uuid
from oslo.config import cfg
import six
import six.moves.urllib.parse as urlparse
import stubout
import swiftclient
import glance.common.auth
from glance.common import exception
from glance.common import swift_store_utils
from glance.common import utils
from glance.openstack.common import units
from glance.store import BackendException
from glance.store.location import get_location_from_uri
from glance.store import swift
from glance.store.swift import swift_retry_iter
from glance.tests.unit import base
CONF = cfg.CONF
FAKE_UUID = lambda: str(uuid.uuid4())
@ -50,11 +54,15 @@ SWIFT_CONF = {'verbose': True,
'debug': True,
'known_stores': ['glance.store.swift.Store'],
'default_store': 'swift',
'swift_store_auth_address': 'localhost:8080',
'swift_store_container': 'glance',
'swift_store_user': 'user',
'swift_store_key': 'key',
'swift_store_auth_address': 'localhost:8080',
'swift_store_container': 'glance',
'swift_store_retry_get_count': 1}
'swift_store_retry_get_count': 1,
'default_swift_reference': 'ref1'
}
# We stub out as little as possible to ensure that the code paths
@ -223,7 +231,7 @@ class SwiftTests(object):
@property
def swift_store_user(self):
return urlparse.quote(CONF.swift_store_user)
return 'tenant:user1'
def test_get_size(self):
"""
@ -235,6 +243,27 @@ class SwiftTests(object):
image_size = self.store.get_size(loc)
self.assertEqual(image_size, 5120)
def test_validate_location_for_invalid_uri(self):
"""
Test that validate location raises when the location contains
any account reference.
"""
uri = "swift+config://store_1/glance/%s"
self.assertRaises(exception.BadStoreUri,
self.store.validate_location,
uri)
def test_validate_location_for_valid_uri(self):
"""
Test that validate location verifies that the location does not
contain any account reference
"""
uri = "swift://user:key@auth_address/glance/%s"
try:
self.assertIsNone(self.store.validate_location(uri))
except Exception:
self.fail('Location uri validation failed')
def test_get_size_with_multi_tenant_on(self):
"""Test that single tenant uris work with multi tenant on."""
uri = ("swift://%s:key@auth_address/glance/%s" %
@ -315,13 +344,16 @@ class SwiftTests(object):
def test_add(self):
"""Test that we can add an image via the swift backend"""
swift_store_utils.is_multiple_swift_store_accounts_enabled = \
mock.Mock(return_value=False)
reload(swift)
self.store = Store()
expected_swift_size = FIVE_KB
expected_swift_contents = "*" * expected_swift_size
expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
expected_image_id = str(uuid.uuid4())
loc = 'swift+https://%s:key@localhost:8080/glance/%s'
expected_location = loc % (self.swift_store_user,
expected_image_id)
loc = "swift+https://tenant%%3Auser1:key@localhost:8080/glance/%s"
expected_location = loc % (expected_image_id)
image_swift = six.StringIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS
@ -345,35 +377,48 @@ class SwiftTests(object):
self.assertEqual(expected_swift_contents, new_image_contents)
self.assertEqual(expected_swift_size, new_image_swift_size)
def test_add_multi_store(self):
conf = copy.deepcopy(SWIFT_CONF)
conf['default_swift_reference'] = 'store_2'
self.config(**conf)
reload(swift)
self.store = Store()
expected_swift_size = FIVE_KB
expected_swift_contents = "*" * expected_swift_size
expected_image_id = str(uuid.uuid4())
image_swift = six.StringIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0
loc = 'swift+config://store_2/glance/%s'
expected_location = loc % (expected_image_id)
location, size, checksum, arg = self.store.add(expected_image_id,
image_swift,
expected_swift_size)
self.assertEqual(expected_location, location)
def test_add_auth_url_variations(self):
"""
Test that we can add an image via the swift backend with
a variety of different auth_address values
"""
swift_store_utils.is_multiple_swift_store_accounts_enabled = \
mock.Mock(return_value=True)
conf = copy.deepcopy(SWIFT_CONF)
self.config(**conf)
variations = {
'http://localhost:80': 'swift+http://%s:key@localhost:80'
'/glance/%s',
'http://localhost': 'swift+http://%s:key@localhost/glance/%s',
'http://localhost/v1': 'swift+http://%s:key@localhost'
'/v1/glance/%s',
'http://localhost/v1/': 'swift+http://%s:key@localhost'
'/v1/glance/%s',
'https://localhost': 'swift+https://%s:key@localhost/glance/%s',
'https://localhost:8080': 'swift+https://%s:key@localhost:8080'
'/glance/%s',
'https://localhost/v1': 'swift+https://%s:key@localhost'
'/v1/glance/%s',
'https://localhost/v1/': 'swift+https://%s:key@localhost'
'/v1/glance/%s',
'localhost': 'swift+https://%s:key@localhost/glance/%s',
'localhost:8080/v1': 'swift+https://%s:key@localhost:8080'
'/v1/glance/%s',
'store_4': 'swift+config://store_4/glance/%s',
'store_5': 'swift+config://store_5/glance/%s',
'store_6': 'swift+config://store_6/glance/%s'
}
for variation, expected_location in variations.items():
image_id = str(uuid.uuid4())
expected_location = expected_location % (
self.swift_store_user, image_id)
expected_location = expected_location % image_id
expected_swift_size = FIVE_KB
expected_swift_contents = "*" * expected_swift_size
expected_checksum = \
@ -383,8 +428,9 @@ class SwiftTests(object):
global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0
self.config(swift_store_auth_address=variation)
conf['default_swift_reference'] = variation
self.config(**conf)
reload(swift)
self.store = Store()
location, size, checksum, _ = self.store.add(image_id, image_swift,
expected_swift_size)
@ -407,8 +453,13 @@ class SwiftTests(object):
Tests that adding an image with a non-existing container
raises an appropriate exception
"""
self.config(swift_store_create_container_on_put=False,
swift_store_container='noexist')
conf = copy.deepcopy(SWIFT_CONF)
conf['swift_store_user'] = 'tenant:user'
conf['swift_store_create_container_on_put'] = False
conf['swift_store_container'] = 'noexist'
self.config(**conf)
reload(swift)
self.store = Store()
image_swift = six.StringIO("nevergonnamakeit")
@ -434,20 +485,24 @@ class SwiftTests(object):
Tests that adding an image with a non-existing container
creates the container automatically if flag is set
"""
swift_store_utils.is_multiple_swift_store_accounts_enabled = \
mock.Mock(return_value=True)
expected_swift_size = FIVE_KB
expected_swift_contents = "*" * expected_swift_size
expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
expected_image_id = str(uuid.uuid4())
loc = 'swift+https://%s:key@localhost:8080/noexist/%s'
expected_location = loc % (self.swift_store_user,
expected_image_id)
loc = 'swift+config://ref1/noexist/%s'
expected_location = loc % (expected_image_id)
image_swift = six.StringIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0
self.config(swift_store_create_container_on_put=True,
swift_store_container='noexist')
conf = copy.deepcopy(SWIFT_CONF)
conf['swift_store_user'] = 'tenant:user'
conf['swift_store_create_container_on_put'] = True
conf['swift_store_container'] = 'noexist'
self.config(**conf)
reload(swift)
self.store = Store()
location, size, checksum, _ = self.store.add(expected_image_id,
image_swift,
@ -473,19 +528,19 @@ class SwiftTests(object):
and then verify that there have been a number of calls to
put_object()...
"""
swift_store_utils.is_multiple_swift_store_accounts_enabled = \
mock.Mock(return_value=True)
expected_swift_size = FIVE_KB
expected_swift_contents = "*" * expected_swift_size
expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
expected_image_id = str(uuid.uuid4())
loc = 'swift+https://%s:key@localhost:8080/glance/%s'
expected_location = loc % (self.swift_store_user,
expected_image_id)
loc = 'swift+config://ref1/glance/%s'
expected_location = loc % (expected_image_id)
image_swift = six.StringIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0
self.config(swift_store_container='glance')
self.store = Store()
orig_max_size = self.store.large_object_size
orig_temp_size = self.store.large_object_chunk_size
@ -530,9 +585,8 @@ class SwiftTests(object):
expected_swift_contents = "*" * expected_swift_size
expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
expected_image_id = str(uuid.uuid4())
loc = 'swift+https://%s:key@localhost:8080/glance/%s'
expected_location = loc % (self.swift_store_user,
expected_image_id)
loc = 'swift+config://ref1/glance/%s'
expected_location = loc % (expected_image_id)
image_swift = six.StringIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS
@ -540,7 +594,7 @@ class SwiftTests(object):
# Temporarily set Swift MAX_SWIFT_OBJECT_SIZE to 1KB and add our image,
# explicitly setting the image_length to 0
self.config(swift_store_container='glance')
self.store = Store()
orig_max_size = self.store.large_object_size
orig_temp_size = self.store.large_object_chunk_size
@ -621,23 +675,25 @@ class SwiftTests(object):
return False
return False
def test_no_user(self):
def test_no_store_credentials(self):
"""
Tests that options without user disables the add method
Tests that options without a valid credentials disables the add method
"""
self.assertTrue(self._option_required('swift_store_user'))
def test_no_key(self):
"""
Tests that options without key disables the add method
"""
self.assertTrue(self._option_required('swift_store_key'))
swift.SWIFT_STORE_REF_PARAMS = {'ref1': {'auth_address':
'authurl.com', 'user': '',
'key': ''}}
self.store = Store()
self.assertEqual(self.store.add, self.store.add_disabled)
def test_no_auth_address(self):
"""
Tests that options without auth address disables the add method
"""
self.assertTrue(self._option_required('swift_store_auth_address'))
swift.SWIFT_STORE_REF_PARAMS = {'ref1': {'auth_address':
'', 'user': 'user1',
'key': 'key1'}}
self.store = Store()
self.assertEqual(self.store.add, self.store.add_disabled)
def test_delete(self):
"""
@ -650,6 +706,16 @@ class SwiftTests(object):
self.assertRaises(exception.NotFound, self.store.get, loc)
def test_delete_with_reference_params(self):
"""
Test we can delete an existing image in the swift store
"""
uri = "swift+config://ref1/glance/%s" % (FAKE_UUID)
loc = get_location_from_uri(uri)
self.store.delete(loc)
self.assertRaises(exception.NotFound, self.store.get, loc)
def test_delete_non_existing(self):
"""
Test that trying to delete a swift that doesn't exist
@ -712,14 +778,20 @@ class TestStoreAuthV1(base.StoreClearingUnitTest, SwiftTests):
def getConfig(self):
conf = SWIFT_CONF.copy()
conf['swift_store_auth_version'] = '1'
conf['swift_store_user'] = 'user'
conf['swift_store_user'] = 'tenant:user1'
return conf
def setUp(self):
"""Establish a clean test environment"""
conf = self.getConfig()
self.config(**conf)
conf_file = 'glance-swift.conf'
self.test_dir = self.useFixture(fixtures.TempDir()).path
self.swift_config_file = self._copy_data_file(conf_file, self.test_dir)
conf.update({'swift_store_config_file': conf_file})
self.config(swift_store_config_file=self.swift_config_file)
super(TestStoreAuthV1, self).setUp()
swift.SWIFT_STORE_REF_PARAMS = swift_store_utils.SwiftParams().params
self.stubs = stubout.StubOutForTesting()
stub_out_swiftclient(self.stubs, conf['swift_store_auth_version'])
self.store = Store()
@ -730,15 +802,12 @@ class TestStoreAuthV2(TestStoreAuthV1):
def getConfig(self):
conf = super(TestStoreAuthV2, self).getConfig()
conf['swift_store_user'] = 'tenant:user'
conf['swift_store_auth_version'] = '2'
conf['swift_store_user'] = 'tenant:user1'
return conf
def test_v2_with_no_tenant(self):
conf = self.getConfig()
conf['swift_store_user'] = 'failme'
uri = "swift://%s:key@auth_address/glance/%s" % (
conf['swift_store_user'], FAKE_UUID)
uri = "swift://failme:key@auth_address/glance/%s" % (FAKE_UUID)
loc = get_location_from_uri(uri)
self.assertRaises(exception.BadStoreUri,
self.store.get,
@ -779,8 +848,8 @@ class TestSingleTenantStoreConnections(base.IsolatedUnitTest):
self.store = glance.store.swift.SingleTenantStore()
specs = {'scheme': 'swift',
'auth_or_store_url': 'example.com/v2/',
'user': 'tenant:user',
'key': 'abcdefg',
'user': 'tenant:user1',
'key': 'key1',
'container': 'cont',
'obj': 'object'}
self.location = glance.store.swift.StoreLocation(specs)
@ -789,10 +858,10 @@ class TestSingleTenantStoreConnections(base.IsolatedUnitTest):
connection = self.store.get_connection(self.location)
self.assertEqual(connection.authurl, 'https://example.com/v2/')
self.assertEqual(connection.auth_version, '2')
self.assertEqual(connection.user, 'user')
self.assertEqual(connection.user, 'user1')
self.assertEqual(connection.tenant_name, 'tenant')
self.assertEqual(connection.key, 'abcdefg')
self.assertFalse(connection.snet)
self.assertEqual(connection.key, 'key1')
self.assertIsNone(connection.preauthurl)
self.assertIsNone(connection.preauthtoken)
self.assertFalse(connection.insecure)
@ -863,13 +932,34 @@ class TestSingleTenantStoreConnections(base.IsolatedUnitTest):
connection = self.store.get_connection(self.location)
self.assertTrue(connection.snet)
def test_bad_location_uri(self):
self.store.configure()
self.location.uri = 'http://bad_uri://'
self.assertRaises(exception.BadStoreUri,
self.location.parse_uri,
self.location.uri)
def test_bad_location_uri_invalid_credentials(self):
self.store.configure()
self.location.uri = 'swift://bad_creds@uri/cont/obj'
self.assertRaises(exception.BadStoreUri,
self.location.parse_uri,
self.location.uri)
def test_bad_location_uri_invalid_object_path(self):
self.store.configure()
self.location.uri = 'swift://user:key@uri/cont'
self.assertRaises(exception.BadStoreUri,
self.location.parse_uri,
self.location.uri)
class TestMultiTenantStoreConnections(base.IsolatedUnitTest):
def setUp(self):
super(TestMultiTenantStoreConnections, self).setUp()
self.stubs.Set(swiftclient, 'Connection', FakeConnection)
self.context = glance.context.RequestContext(
user='user', tenant='tenant', auth_tok='0123')
user='tenant:user1', tenant='tenant', auth_tok='0123')
self.store = glance.store.swift.MultiTenantStore(self.context)
specs = {'scheme': 'swift',
'auth_or_store_url': 'example.com',
@ -882,7 +972,7 @@ class TestMultiTenantStoreConnections(base.IsolatedUnitTest):
connection = self.store.get_connection(self.location)
self.assertIsNone(connection.authurl)
self.assertEqual(connection.auth_version, '2')
self.assertEqual(connection.user, 'user')
self.assertEqual(connection.user, 'tenant:user1')
self.assertEqual(connection.tenant_name, 'tenant')
self.assertIsNone(connection.key)
self.assertFalse(connection.snet)
@ -910,29 +1000,44 @@ class FakeGetEndpoint(object):
class TestCreatingLocations(base.IsolatedUnitTest):
def setUp(self):
conf = copy.deepcopy(SWIFT_CONF)
self.config(**conf)
reload(swift)
super(TestCreatingLocations, self).setUp()
def test_single_tenant_location(self):
self.config(swift_store_auth_address='example.com/v2',
swift_store_container='container',
swift_store_user='tenant:user',
swift_store_key='auth_key')
store = glance.store.swift.SingleTenantStore()
conf = copy.deepcopy(SWIFT_CONF)
conf['swift_store_container'] = 'container'
conf_file = "glance-swift.conf"
test_dir = self.useFixture(fixtures.TempDir()).path
self.swift_config_file = self._copy_data_file(conf_file, test_dir)
conf.update({'swift_store_config_file': self.swift_config_file})
conf['default_swift_reference'] = 'ref1'
self.config(**conf)
reload(swift)
store = swift.SingleTenantStore()
location = store.create_location('image-id')
self.assertEqual(location.scheme, 'swift+https')
self.assertEqual(location.swift_url, 'https://example.com/v2')
self.assertEqual(location.swift_url, 'https://example.com')
self.assertEqual(location.container, 'container')
self.assertEqual(location.obj, 'image-id')
self.assertEqual(location.user, 'tenant:user')
self.assertEqual(location.key, 'auth_key')
self.assertEqual(location.user, 'tenant:user1')
self.assertEqual(location.key, 'key1')
def test_single_tenant_location_http(self):
self.config(swift_store_auth_address='http://example.com/v2',
swift_store_container='container',
swift_store_user='tenant:user',
swift_store_key='auth_key')
conf_file = "glance-swift.conf"
test_dir = self.useFixture(fixtures.TempDir()).path
self.swift_config_file = self._copy_data_file(conf_file, test_dir)
self.config(swift_store_container='container',
default_swift_reference='ref2',
swift_store_config_file=self.swift_config_file)
swift.SWIFT_STORE_REF_PARAMS = swift_store_utils.SwiftParams().params
store = glance.store.swift.SingleTenantStore()
location = store.create_location('image-id')
self.assertEqual(location.scheme, 'swift+http')
self.assertEqual(location.swift_url, 'http://example.com/v2')
self.assertEqual(location.swift_url, 'http://example.com')
def test_multi_tenant_location(self):
self.config(swift_store_container='container')
@ -994,6 +1099,10 @@ class TestCreatingLocations(base.IsolatedUnitTest):
class TestChunkReader(base.StoreClearingUnitTest):
def setUp(self):
conf = copy.deepcopy(SWIFT_CONF)
self.config(**conf)
super(TestChunkReader, self).setUp()
def test_read_all_data(self):
"""

23
glance/tests/unit/v1/test_api.py

@ -91,6 +91,7 @@ class TestGlanceAPI(base.IsolatedUnitTest):
'metadata': {}}],
'properties': {}}]
self.context = glance.context.RequestContext(is_admin=True)
glance.api.v1.images.validate_location = mock.Mock()
db_api.get_engine()
self.destroy_fixtures()
self.create_fixtures()
@ -959,6 +960,28 @@ class TestGlanceAPI(base.IsolatedUnitTest):
res = req.get_response(self.api)
self.assertEqual(res.status_int, 409)
def test_add_location_with_invalid_location(self):
"""Tests creates an image from location and conflict image size"""
mock_validate_location = mock.Mock()
glance.api.v1.images.validate_location = mock_validate_location
mock_validate_location.side_effect = exception.BadStoreUri()
fixture_headers = {'x-image-meta-store': 'file',
'x-image-meta-disk-format': 'vhd',
'x-image-meta-location': 'http://a/b/c.tar.gz',
'x-image-meta-container-format': 'ovf',
'x-image-meta-name': 'fake image #F',
'x-image-meta-size': '1'}
req = webob.Request.blank("/images")
req.headers['Content-Type'] = 'application/octet-stream'
req.method = 'POST'
for k, v in fixture_headers.iteritems():
req.headers[k] = v
res = req.get_response(self.api)
self.assertEqual(res.status_int, 400)
def test_add_copy_from_with_location(self):
"""Tests creates an image from copy-from and location"""
fixture_headers = {'x-image-meta-store': 'file',