From f60d05686ffa8d74a9018a5bc94cb618091f4c6c Mon Sep 17 00:00:00 2001 From: gholt Date: Sun, 8 Dec 2013 09:13:59 +0000 Subject: [PATCH] New container sync configuration option Summary of the new configuration option: The cluster operators add the container_sync middleware to their proxy pipeline and create a container-sync-realms.conf for their cluster and copy this out to all their proxy and container servers. This file specifies the available container sync "realms". A container sync realm is a group of clusters with a shared key that have agreed to provide container syncing to one another. The end user can then set the X-Container-Sync-To value on a container to //realm/cluster/account/container instead of the previously required URL. The allowed hosts list is not used with this configuration and instead every container sync request sent is signed using the realm key and user key. This offers better security as source hosts can be faked much more easily than faking per request signatures. Replaying signed requests, assuming it could easily be done, shouldn't be an issue as the X-Timestamp is part of the signature and so would just short-circuit as already current or as superceded. This also makes configuration easier for the end user, especially with difficult networking situations where a different host might need to be used for the container sync daemon since it's connecting from within a cluster. With this new configuration option, the end user just specifies the realm and cluster names and that is resolved to the proper endpoint configured by the operator. If the operator changes their configuration (key or endpoint), the end user does not need to change theirs. DocImpact Change-Id: Ie1704990b66d0434e4991e26ed1da8b08cb05a37 --- doc/source/misc.rst | 14 ++ doc/source/overview_container_sync.rst | 214 ++++++++++++++-- etc/container-server.conf-sample | 4 +- etc/container-sync-realms.conf-sample | 47 ++++ etc/proxy-server.conf-sample | 11 +- setup.cfg | 1 + swift/common/container_sync_realms.py | 159 ++++++++++++ swift/common/middleware/container_sync.py | 122 +++++++++ swift/common/utils.py | 63 ++++- swift/container/server.py | 19 +- swift/container/sync.py | 73 ++++-- .../common/middleware/test_container_sync.py | 233 ++++++++++++++++++ .../unit/common/test_container_sync_realms.py | 187 ++++++++++++++ test/unit/common/test_utils.py | 122 +++++++-- test/unit/container/test_sync.py | 82 ++++-- 15 files changed, 1262 insertions(+), 89 deletions(-) create mode 100644 etc/container-sync-realms.conf-sample create mode 100644 swift/common/container_sync_realms.py create mode 100644 swift/common/middleware/container_sync.py create mode 100644 test/unit/common/middleware/test_container_sync.py create mode 100644 test/unit/common/test_container_sync_realms.py diff --git a/doc/source/misc.rst b/doc/source/misc.rst index 3aea382458..4e201f2134 100644 --- a/doc/source/misc.rst +++ b/doc/source/misc.rst @@ -217,6 +217,20 @@ List Endpoints :members: :show-inheritance: +Container Sync Realms +===================== + +.. automodule:: swift.common.container_sync_realms + :members: + :show-inheritance: + +Container Sync Middleware +========================= + +.. automodule:: swift.common.middleware.container_sync + :members: + :show-inheritance: + Discoverability =============== diff --git a/doc/source/overview_container_sync.rst b/doc/source/overview_container_sync.rst index c0ab3a10b6..1485368d8f 100644 --- a/doc/source/overview_container_sync.rst +++ b/doc/source/overview_container_sync.rst @@ -25,13 +25,76 @@ synchronization key. your manifest file and your segment files are synced if they happen to be in different containers. --------------------------------------------- -Configuring a Cluster's Allowable Sync Hosts --------------------------------------------- +-------------------------- +Configuring Container Sync +-------------------------- -The Swift cluster operator must allow synchronization with a set of hosts -before the user can enable container synchronization. First, the backend -container server needs to be given this list of hosts in the +Create a container-sync-realms.conf file specifying the allowable clusters +and their information:: + + [realm1] + key = realm1key + key2 = realm1key2 + cluster_name1 = https://host1/v1/ + cluster_name2 = https://host2/v1/ + + [realm2] + key = realm2key + key2 = realm2key2 + cluster_name3 = https://host3/v1/ + cluster_name4 = https://host4/v1/ + + +Each section name is the name of a sync realm. A sync realm is a set of +clusters that have agreed to allow container syncing with each other. Realm +names will be considered case insensitive. + +The key is the overall cluster-to-cluster key used in combination with the +external users' key that they set on their containers' X-Container-Sync-Key +metadata header values. These keys will be used to sign each request the +container sync daemon makes and used to validate each incoming container sync +request. + +The key2 is optional and is an additional key incoming requests will be checked +against. This is so you can rotate keys if you wish; you move the existing key +to key2 and make a new key value. + +Any values in the realm section whose names begin with cluster\_ will indicate +the name and endpoint of a cluster and will be used by external users in +their containers' X-Container-Sync-To metadata header values with the format +"//realm_name/cluster_name/account_name/container_name". Realm and cluster +names are considered case insensitive. + +The endpoint is what the container sync daemon will use when sending out +requests to that cluster. Keep in mind this endpoint must be reachable by all +container servers, since that is where the container sync daemon runs. Note +that the endpoint ends with /v1/ and that the container sync daemon will then +add the account/container/obj name after that. + +Distribute this container-sync-realms.conf file to all your proxy servers +and container servers. + +You also need to add the container_sync middleware to your proxy pipeline. It +needs to be after any memcache middleware and before any auth middleware. The +container_sync section only needs the "use" item. For example:: + + [pipeline:main] + pipeline = healthcheck proxy-logging cache container_sync tempauth proxy-logging proxy-server + + [filter:container_sync] + use = egg:swift#container_sync + + +------------------------------------------------------- +Old-Style: Configuring a Cluster's Allowable Sync Hosts +------------------------------------------------------- + +This section is for the old-style of using container sync. See the previous +section, Configuring Container Sync, for the new-style. + +With the old-style, the Swift cluster operator must allow synchronization with +a set of hosts before the user can enable container synchronization. First, the +backend container server needs to be given this list of hosts in the container-server.conf file:: [DEFAULT] @@ -52,13 +115,18 @@ container-server.conf file:: # Maximum amount of time to spend syncing each container # container_time = 60 + +---------------------- +Logging Container Sync +---------------------- + Tracking sync progress, problems, and just general activity can only be -achieved with log processing for this first release of container -synchronization. In that light, you may wish to set the above `log_` options to -direct the container-sync logs to a different file for easier monitoring. -Additionally, it should be noted there is no way for an end user to detect sync -progress or problems other than HEADing both containers and comparing the -overall information. +achieved with log processing currently for container synchronization. In that +light, you may wish to set the above `log_` options to direct the +container-sync logs to a different file for easier monitoring. Additionally, it +should be noted there is no way for an end user to detect sync progress or +problems other than HEADing both containers and comparing the overall +information. ---------------------------------------------------------- Using the ``swift`` tool to set up synchronized containers @@ -73,6 +141,112 @@ Using the ``swift`` tool to set up synchronized containers You must be the account admin on the account to set synchronization targets and keys. +You simply tell each container where to sync to and give it a secret +synchronization key. First, let's get the account details for our two cluster +accounts:: + + $ swift -A http://cluster1/auth/v1.0 -U test:tester -K testing stat -v + StorageURL: http://cluster1/v1/AUTH_208d1854-e475-4500-b315-81de645d060e + Auth Token: AUTH_tkd5359e46ff9e419fa193dbd367f3cd19 + Account: AUTH_208d1854-e475-4500-b315-81de645d060e + Containers: 0 + Objects: 0 + Bytes: 0 + + $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 stat -v + StorageURL: http://cluster2/v1/AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c + Auth Token: AUTH_tk816a1aaf403c49adb92ecfca2f88e430 + Account: AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c + Containers: 0 + Objects: 0 + Bytes: 0 + +Now, let's make our first container and tell it to synchronize to a second +we'll make next:: + + $ swift -A http://cluster1/auth/v1.0 -U test:tester -K testing post \ + -t '//realm_name/cluster2_name/AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c/container2' \ + -k 'secret' container1 + +The ``-t`` indicates the cluster to sync to, which is the realm name of the +section from container-sync-realms.conf, followed by the cluster name from +that section, followed by the account and container names we want to sync to. +The ``-k`` specifies the secret key the two containers will share for +synchronization; this is the user key, the cluster key in +container-sync-realms.conf will also be used behind the scenes. + +Now, we'll do something similar for the second cluster's container:: + + $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 post \ + -t '//realm_name/cluster1_name/AUTH_208d1854-e475-4500-b315-81de645d060e/container1' \ + -k 'secret' container2 + +That's it. Now we can upload a bunch of stuff to the first container and watch +as it gets synchronized over to the second:: + + $ swift -A http://cluster1/auth/v1.0 -U test:tester -K testing \ + upload container1 . + photo002.png + photo004.png + photo001.png + photo003.png + + $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 \ + list container2 + + [Nothing there yet, so we wait a bit...] + [If you're an operator running SAIO and just testing, you may need to + run 'swift-init container-sync once' to perform a sync scan.] + + $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 \ + list container2 + photo001.png + photo002.png + photo003.png + photo004.png + +You can also set up a chain of synced containers if you want more than two. +You'd point 1 -> 2, then 2 -> 3, and finally 3 -> 1 for three containers. +They'd all need to share the same secret synchronization key. + +.. _`python-swiftclient`: http://github.com/openstack/python-swiftclient + +----------------------------------- +Using curl (or other tools) instead +----------------------------------- + +So what's ``swift`` doing behind the scenes? Nothing overly complicated. It +translates the ``-t `` option into an ``X-Container-Sync-To: `` +header and the ``-k `` option into an ``X-Container-Sync-Key: `` +header. + +For instance, when we created the first container above and told it to +synchronize to the second, we could have used this curl command:: + + $ curl -i -X POST -H 'X-Auth-Token: AUTH_tkd5359e46ff9e419fa193dbd367f3cd19' \ + -H 'X-Container-Sync-To: //realm_name/cluster2_name/AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c/container2' \ + -H 'X-Container-Sync-Key: secret' \ + 'http://cluster1/v1/AUTH_208d1854-e475-4500-b315-81de645d060e/container1' + HTTP/1.1 204 No Content + Content-Length: 0 + Content-Type: text/plain; charset=UTF-8 + Date: Thu, 24 Feb 2011 22:39:14 GMT + +--------------------------------------------------------------------- +Old-Style: Using the ``swift`` tool to set up synchronized containers +--------------------------------------------------------------------- + +.. note:: + + The ``swift`` tool is available from the `python-swiftclient`_ library. + +.. note:: + + You must be the account admin on the account to set synchronization targets + and keys. + +This is for the old-style of container syncing using allowed_sync_hosts. + You simply tell each container where to sync to and give it a secret synchronization key. First, let's get the account details for our two cluster accounts:: @@ -139,9 +313,11 @@ They'd all need to share the same secret synchronization key. .. _`python-swiftclient`: http://github.com/openstack/python-swiftclient ------------------------------------ -Using curl (or other tools) instead ------------------------------------ +---------------------------------------------- +Old-Style: Using curl (or other tools) instead +---------------------------------------------- + +This is for the old-style of container syncing using allowed_sync_hosts. So what's ``swift`` doing behind the scenes? Nothing overly complicated. It translates the ``-t `` option into an ``X-Container-Sync-To: `` @@ -174,10 +350,10 @@ to the other container. .. note:: - The swift-container-sync process runs on each container server in - the cluster and talks to the proxy servers in the remote cluster. - Therefore, the container servers must be permitted to initiate - outbound connections to the remote proxy servers. + The swift-container-sync process runs on each container server in the + cluster and talks to the proxy servers (or load balancers) in the remote + cluster. Therefore, the container servers must be permitted to initiate + outbound connections to the remote proxy servers (or load balancers). .. note:: diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index e4c15907e0..da56b03ff0 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -17,7 +17,9 @@ # max_clients = 1024 # # This is a comma separated list of hosts allowed in the X-Container-Sync-To -# field for containers. +# field for containers. This is the old-style of using container sync. It is +# strongly recommended to use the new style of a separate +# container-sync-realms.conf -- see container-sync-realms.conf-sample # allowed_sync_hosts = 127.0.0.1 # # You can specify default log routing here if you want: diff --git a/etc/container-sync-realms.conf-sample b/etc/container-sync-realms.conf-sample new file mode 100644 index 0000000000..1eaddc19b3 --- /dev/null +++ b/etc/container-sync-realms.conf-sample @@ -0,0 +1,47 @@ +# [DEFAULT] +# The number of seconds between checking the modified time of this config file +# for changes and therefore reloading it. +# mtime_check_interval = 300 + + +# [realm1] +# key = realm1key +# key2 = realm1key2 +# cluster_name1 = https://host1/v1/ +# cluster_name2 = https://host2/v1/ +# +# [realm2] +# key = realm2key +# key2 = realm2key2 +# cluster_name3 = https://host3/v1/ +# cluster_name4 = https://host4/v1/ + + +# Each section name is the name of a sync realm. A sync realm is a set of +# clusters that have agreed to allow container syncing with each other. Realm +# names will be considered case insensitive. +# +# The key is the overall cluster-to-cluster key used in combination with the +# external users' key that they set on their containers' X-Container-Sync-Key +# metadata header values. These keys will be used to sign each request the +# container sync daemon makes and used to validate each incoming container sync +# request. +# +# The key2 is optional and is an additional key incoming requests will be +# checked against. This is so you can rotate keys if you wish; you move the +# existing key to key2 and make a new key value. +# +# Any values in the realm section whose names begin with cluster_ will indicate +# the name and endpoint of a cluster and will be used by external users in +# their containers' X-Container-Sync-To metadata header values with the format +# "realm_name/cluster_name/container_name". Realm and cluster names are +# considered case insensitive. +# +# The endpoint is what the container sync daemon will use when sending out +# requests to that cluster. Keep in mind this endpoint must be reachable by all +# container servers, since that is where the container sync daemon runs. Note +# the the endpoint ends with /v1/ and that the container sync daemon will then +# add the account/container/obj name after that. +# +# Distribute this container-sync-realms.conf file to all your proxy servers +# and container servers. diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 4f9260476c..78d944f457 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -69,7 +69,7 @@ # eventlet_debug = false [pipeline:main] -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk slo ratelimit tempauth container-quotas account-quotas proxy-logging proxy-server +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk slo ratelimit tempauth container-quotas account-quotas proxy-logging proxy-server [app:proxy-server] use = egg:swift#proxy @@ -526,3 +526,12 @@ use = egg:swift#gatekeeper # set log_level = INFO # set log_headers = false # set log_address = /dev/log + +[filter:container_sync] +use = egg:swift#container_sync +# Set this to false if you want to disallow any full url values to be set for +# any new X-Container-Sync-To headers. This will keep any new full urls from +# coming in, but won't change any existing values already in the cluster. +# Updating those will have to be done manually, as knowing what the true realm +# endpoint should be cannot always be guessed. +# allow_full_urls = true diff --git a/setup.cfg b/setup.cfg index 1102b86502..0b7cabfac6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -87,6 +87,7 @@ paste.filter_factory = slo = swift.common.middleware.slo:filter_factory list_endpoints = swift.common.middleware.list_endpoints:filter_factory gatekeeper = swift.common.middleware.gatekeeper:filter_factory + container_sync = swift.common.middleware.container_sync:filter_factory [build_sphinx] all_files = 1 diff --git a/swift/common/container_sync_realms.py b/swift/common/container_sync_realms.py new file mode 100644 index 0000000000..083c5e1fd9 --- /dev/null +++ b/swift/common/container_sync_realms.py @@ -0,0 +1,159 @@ +# Copyright (c) 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ConfigParser +import errno +import hashlib +import hmac +import os +import time + +from swift import gettext_ as _ +from swift.common.utils import get_valid_utf8_str + + +class ContainerSyncRealms(object): + """ + Loads and parses the container-sync-realms.conf, occasionally + checking the file's mtime to see if it needs to be reloaded. + """ + + def __init__(self, conf_path, logger): + self.conf_path = conf_path + self.logger = logger + self.next_mtime_check = 0 + self.mtime_check_interval = 300 + self.conf_path_mtime = 0 + self.data = {} + self.reload() + + def reload(self): + """Forces a reload of the conf file.""" + self.next_mtime_check = 0 + self.conf_path_mtime = 0 + self._reload() + + def _reload(self): + now = time.time() + if now >= self.next_mtime_check: + self.next_mtime_check = now + self.mtime_check_interval + try: + mtime = os.path.getmtime(self.conf_path) + except OSError as err: + if err.errno == errno.ENOENT: + log_func = self.logger.debug + else: + log_func = self.logger.error + log_func(_('Could not load %r: %s'), self.conf_path, err) + else: + if mtime != self.conf_path_mtime: + self.conf_path_mtime = mtime + try: + conf = ConfigParser.SafeConfigParser() + conf.read(self.conf_path) + except ConfigParser.ParsingError as err: + self.logger.error( + _('Could not load %r: %s'), self.conf_path, err) + else: + try: + self.mtime_check_interval = conf.getint( + 'DEFAULT', 'mtime_check_interval') + self.next_mtime_check = \ + now + self.mtime_check_interval + except ConfigParser.NoOptionError: + self.mtime_check_interval = 300 + self.next_mtime_check = \ + now + self.mtime_check_interval + except (ConfigParser.ParsingError, ValueError) as err: + self.logger.error( + _('Error in %r with mtime_check_interval: %s'), + self.conf_path, err) + realms = {} + for section in conf.sections(): + realm = {} + clusters = {} + for option, value in conf.items(section): + if option in ('key', 'key2'): + realm[option] = value + elif option.startswith('cluster_'): + clusters[option[8:].upper()] = value + realm['clusters'] = clusters + realms[section.upper()] = realm + self.data = realms + + def realms(self): + """Returns a list of realms.""" + self._reload() + return self.data.keys() + + def key(self, realm): + """Returns the key for the realm.""" + self._reload() + result = self.data.get(realm.upper()) + if result: + result = result.get('key') + return result + + def key2(self, realm): + """Returns the key2 for the realm.""" + self._reload() + result = self.data.get(realm.upper()) + if result: + result = result.get('key2') + return result + + def clusters(self, realm): + """Returns a list of clusters for the realm.""" + self._reload() + result = self.data.get(realm.upper()) + if result: + result = result.get('clusters') + if result: + result = result.keys() + return result or [] + + def endpoint(self, realm, cluster): + """Returns the endpoint for the cluster in the realm.""" + self._reload() + result = None + realm_data = self.data.get(realm.upper()) + if realm_data: + cluster_data = realm_data.get('clusters') + if cluster_data: + result = cluster_data.get(cluster.upper()) + return result + + def get_sig(self, request_method, path, x_timestamp, nonce, realm_key, + user_key): + """ + Returns the hexdigest string of the HMAC-SHA1 (RFC 2104) for + the information given. + + :param request_method: HTTP method of the request. + :param path: The path to the resource. + :param x_timestamp: The X-Timestamp header value for the request. + :param nonce: A unique value for the request. + :param realm_key: Shared secret at the cluster operator level. + :param user_key: Shared secret at the user's container level. + :returns: hexdigest str of the HMAC-SHA1 for the request. + """ + nonce = get_valid_utf8_str(nonce) + realm_key = get_valid_utf8_str(realm_key) + user_key = get_valid_utf8_str(user_key) + return hmac.new( + realm_key, + '%s\n%s\n%s\n%s\n%s' % ( + request_method, path, x_timestamp, nonce, user_key), + hashlib.sha1).hexdigest() diff --git a/swift/common/middleware/container_sync.py b/swift/common/middleware/container_sync.py new file mode 100644 index 0000000000..c5393df4fa --- /dev/null +++ b/swift/common/middleware/container_sync.py @@ -0,0 +1,122 @@ +# Copyright (c) 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from swift.common.container_sync_realms import ContainerSyncRealms +from swift.common.swob import HTTPBadRequest, HTTPUnauthorized, wsgify +from swift.common.utils import ( + config_true_value, get_logger, register_swift_info, streq_const_time) +from swift.proxy.controllers.base import get_container_info + + +class ContainerSync(object): + """ + WSGI middleware that validates an incoming container sync request + using the container-sync-realms.conf style of container sync. + """ + + def __init__(self, app, conf): + self.app = app + self.conf = conf + self.logger = get_logger(conf, log_route='container_sync') + self.realms_conf = ContainerSyncRealms( + os.path.join( + conf.get('swift_dir', '/etc/swift'), + 'container-sync-realms.conf'), + self.logger) + self.allow_full_urls = config_true_value( + conf.get('allow_full_urls', 'true')) + + @wsgify + def __call__(self, req): + if not self.allow_full_urls: + sync_to = req.headers.get('x-container-sync-to') + if sync_to and not sync_to.startswith('//'): + raise HTTPBadRequest( + body='Full URLs are not allowed for X-Container-Sync-To ' + 'values. Only realm values of the format ' + '//realm/cluster/account/container are allowed.\n', + request=req) + auth = req.headers.get('x-container-sync-auth') + if auth: + valid = False + auth = auth.split() + if len(auth) != 3: + req.environ.setdefault('swift.log_info', []).append( + 'cs:not-3-args') + else: + realm, nonce, sig = auth + realm_key = self.realms_conf.key(realm) + realm_key2 = self.realms_conf.key2(realm) + if not realm_key: + req.environ.setdefault('swift.log_info', []).append( + 'cs:no-local-realm-key') + else: + info = get_container_info( + req.environ, self.app, swift_source='CS') + user_key = info.get('sync_key') + if not user_key: + req.environ.setdefault('swift.log_info', []).append( + 'cs:no-local-user-key') + else: + expected = self.realms_conf.get_sig( + req.method, req.path, + req.headers.get('x-timestamp', '0'), nonce, + realm_key, user_key) + expected2 = self.realms_conf.get_sig( + req.method, req.path, + req.headers.get('x-timestamp', '0'), nonce, + realm_key2, user_key) if realm_key2 else expected + if not streq_const_time(sig, expected) and \ + not streq_const_time(sig, expected2): + req.environ.setdefault( + 'swift.log_info', []).append('cs:invalid-sig') + else: + req.environ.setdefault( + 'swift.log_info', []).append('cs:valid') + valid = True + if not valid: + exc = HTTPUnauthorized( + body='X-Container-Sync-Auth header not valid; ' + 'contact cluster operator for support.', + headers={'content-type': 'text/plain'}, + request=req) + exc.headers['www-authenticate'] = ' '.join([ + 'SwiftContainerSync', + exc.www_authenticate().split(None, 1)[1]]) + raise exc + else: + req.environ['swift.authorize_override'] = True + if req.path == '/info': + # Ensure /info requests get the freshest results + dct = {} + for realm in self.realms_conf.realms(): + clusters = self.realms_conf.clusters(realm) + if clusters: + dct[realm] = {'clusters': dict((c, {}) for c in clusters)} + register_swift_info('container_sync', realms=dct) + return self.app + + +def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + register_swift_info('container_sync') + + def cache_filter(app): + return ContainerSync(app, conf) + + return cache_filter diff --git a/swift/common/utils.py b/swift/common/utils.py index f19fd7a48d..45ad2bf1fc 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -1765,21 +1765,66 @@ def urlparse(url): return ModifiedParseResult(*stdlib_urlparse(url)) -def validate_sync_to(value, allowed_sync_hosts): +def validate_sync_to(value, allowed_sync_hosts, realms_conf): + """ + Validates an X-Container-Sync-To header value, returning the + validated endpoint, realm, and realm_key, or an error string. + + :param value: The X-Container-Sync-To header value to validate. + :param allowed_sync_hosts: A list of allowed hosts in endpoints, + if realms_conf does not apply. + :param realms_conf: A instance of + swift.common.container_sync_realms.ContainerSyncRealms to + validate against. + :returns: A tuple of (error_string, validated_endpoint, realm, + realm_key). The error_string will None if the rest of the + values have been validated. The validated_endpoint will be + the validated endpoint to sync to. The realm and realm_key + will be set if validation was done through realms_conf. + """ + orig_value = value + value = value.rstrip('/') if not value: - return None + return (None, None, None, None) + if value.startswith('//'): + if not realms_conf: + return (None, None, None, None) + data = value[2:].split('/') + if len(data) != 4: + return ( + _('Invalid X-Container-Sync-To format %r') % orig_value, + None, None, None) + realm, cluster, account, container = data + realm_key = realms_conf.key(realm) + if not realm_key: + return (_('No realm key for %r') % realm, None, None, None) + endpoint = realms_conf.endpoint(realm, cluster) + if not endpoint: + return ( + _('No cluster endpoint for %r %r') % (realm, cluster), + None, None, None) + return ( + None, + '%s/%s/%s' % (endpoint.rstrip('/'), account, container), + realm.upper(), realm_key) p = urlparse(value) if p.scheme not in ('http', 'https'): - return _('Invalid scheme %r in X-Container-Sync-To, must be "http" ' - 'or "https".') % p.scheme + return ( + _('Invalid scheme %r in X-Container-Sync-To, must be "//", ' + '"http", or "https".') % p.scheme, + None, None, None) if not p.path: - return _('Path required in X-Container-Sync-To') + return (_('Path required in X-Container-Sync-To'), None, None, None) if p.params or p.query or p.fragment: - return _('Params, queries, and fragments not allowed in ' - 'X-Container-Sync-To') + return ( + _('Params, queries, and fragments not allowed in ' + 'X-Container-Sync-To'), + None, None, None) if p.hostname not in allowed_sync_hosts: - return _('Invalid host %r in X-Container-Sync-To') % p.hostname - return None + return ( + _('Invalid host %r in X-Container-Sync-To') % p.hostname, + None, None, None) + return (None, value, None, None) def affinity_key_function(affinity_str): diff --git a/swift/container/server.py b/swift/container/server.py index 744e50e33b..e6a3dc7b68 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -25,6 +25,7 @@ from eventlet import Timeout import swift.common.db from swift.container.backend import ContainerBroker from swift.common.db import DatabaseAlreadyExists +from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path, is_sys_or_user_meta from swift.common.utils import get_logger, hash_path, public, \ @@ -62,6 +63,14 @@ class ContainerController(object): if replication_server is not None: replication_server = config_true_value(replication_server) self.replication_server = replication_server + #: ContainerSyncCluster instance for validating sync-to values. + self.realms_conf = ContainerSyncRealms( + os.path.join( + conf.get('swift_dir', '/etc/swift'), + 'container-sync-realms.conf'), + self.logger) + #: The list of hosts we're allowed to send syncs to. This can be + #: overridden by data in self.realms_conf self.allowed_sync_hosts = [ h.strip() for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',') @@ -228,8 +237,9 @@ class ContainerController(object): return HTTPBadRequest(body='Missing timestamp', request=req, content_type='text/plain') if 'x-container-sync-to' in req.headers: - err = validate_sync_to(req.headers['x-container-sync-to'], - self.allowed_sync_hosts) + err, sync_to, realm, realm_key = validate_sync_to( + req.headers['x-container-sync-to'], self.allowed_sync_hosts, + self.realms_conf) if err: return HTTPBadRequest(err) if self.mount_check and not check_mount(self.root, drive): @@ -438,8 +448,9 @@ class ContainerController(object): return HTTPBadRequest(body='Missing or bad timestamp', request=req, content_type='text/plain') if 'x-container-sync-to' in req.headers: - err = validate_sync_to(req.headers['x-container-sync-to'], - self.allowed_sync_hosts) + err, sync_to, realm, realm_key = validate_sync_to( + req.headers['x-container-sync-to'], self.allowed_sync_hosts, + self.realms_conf) if err: return HTTPBadRequest(err) if self.mount_check and not check_mount(self.root, drive): diff --git a/swift/container/sync.py b/swift/container/sync.py index 048efe3fb1..402f602dfd 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import uuid from swift import gettext_ as _ from time import ctime, time from random import random, shuffle @@ -24,11 +26,13 @@ import swift.common.db from swift.container import server as container_server from swiftclient import delete_object, put_object, quote from swift.container.backend import ContainerBroker +from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.direct_client import direct_get_object from swift.common.exceptions import ClientException from swift.common.ring import Ring from swift.common.utils import audit_location_generator, get_logger, \ - hash_path, config_true_value, validate_sync_to, whataremyips, FileLikeIter + hash_path, config_true_value, validate_sync_to, whataremyips, \ + FileLikeIter, urlparse from swift.common.daemon import Daemon from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND @@ -117,7 +121,14 @@ class ContainerSync(Daemon): #: to the next one. If a conatiner sync hasn't finished in this time, #: it'll just be resumed next scan. self.container_time = int(conf.get('container_time', 60)) - #: The list of hosts we're allowed to send syncs to. + #: ContainerSyncCluster instance for validating sync-to values. + self.realms_conf = ContainerSyncRealms( + os.path.join( + conf.get('swift_dir', '/etc/swift'), + 'container-sync-realms.conf'), + self.logger) + #: The list of hosts we're allowed to send syncs to. This can be + #: overridden by data in self.realms_conf self.allowed_sync_hosts = [ h.strip() for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',') @@ -228,20 +239,20 @@ class ContainerSync(Daemon): return if not broker.is_deleted(): sync_to = None - sync_key = None + user_key = None sync_point1 = info['x_container_sync_point1'] sync_point2 = info['x_container_sync_point2'] for key, (value, timestamp) in broker.metadata.iteritems(): if key.lower() == 'x-container-sync-to': sync_to = value elif key.lower() == 'x-container-sync-key': - sync_key = value - if not sync_to or not sync_key: + user_key = value + if not sync_to or not user_key: self.container_skips += 1 self.logger.increment('skips') return - sync_to = sync_to.rstrip('/') - err = validate_sync_to(sync_to, self.allowed_sync_hosts) + err, sync_to, realm, realm_key = validate_sync_to( + sync_to, self.allowed_sync_hosts, self.realms_conf) if err: self.logger.info( _('ERROR %(db_file)s: %(validate_sync_to_err)s'), @@ -267,8 +278,9 @@ class ContainerSync(Daemon): # This section will attempt to sync previously skipped # rows in case the previous attempts by any of the nodes # didn't succeed. - if not self.container_sync_row(row, sync_to, sync_key, - broker, info): + if not self.container_sync_row( + row, sync_to, user_key, broker, info, realm, + realm_key): if not next_sync_point: next_sync_point = sync_point2 sync_point2 = row['ROWID'] @@ -289,8 +301,9 @@ class ContainerSync(Daemon): # succeed or in case it failed to do so the first time. if unpack_from('>I', key)[0] % \ len(nodes) == ordinal: - self.container_sync_row(row, sync_to, sync_key, - broker, info) + self.container_sync_row( + row, sync_to, user_key, broker, info, realm, + realm_key) sync_point1 = row['ROWID'] broker.set_x_container_sync_points(sync_point1, None) self.container_syncs += 1 @@ -301,27 +314,44 @@ class ContainerSync(Daemon): self.logger.exception(_('ERROR Syncing %s'), broker if broker else path) - def container_sync_row(self, row, sync_to, sync_key, broker, info): + def container_sync_row(self, row, sync_to, user_key, broker, info, + realm, realm_key): """ Sends the update the row indicates to the sync_to container. :param row: The updated row in the local database triggering the sync update. :param sync_to: The URL to the remote container. - :param sync_key: The X-Container-Sync-Key to use when sending requests + :param user_key: The X-Container-Sync-Key to use when sending requests to the other container. :param broker: The local container database broker. :param info: The get_info result from the local container database broker. + :param realm: The realm from self.realms_conf, if there is one. + If None, fallback to using the older allowed_sync_hosts + way of syncing. + :param realm_key: The realm key from self.realms_conf, if there + is one. If None, fallback to using the older + allowed_sync_hosts way of syncing. :returns: True on success """ try: start_time = time() if row['deleted']: try: - delete_object(sync_to, name=row['name'], - headers={'x-timestamp': row['created_at'], - 'x-container-sync-key': sync_key}, + headers = {'x-timestamp': row['created_at']} + if realm and realm_key: + nonce = uuid.uuid4().hex + path = urlparse(sync_to).path + '/' + quote( + row['name']) + sig = self.realms_conf.get_sig( + 'DELETE', path, headers['x-timestamp'], nonce, + realm_key, user_key) + headers['x-container-sync-auth'] = '%s %s %s' % ( + realm, nonce, sig) + else: + headers['x-container-sync-key'] = user_key + delete_object(sync_to, name=row['name'], headers=headers, proxy=self.proxy) except ClientException as err: if err.http_status != HTTP_NOT_FOUND: @@ -373,7 +403,16 @@ class ContainerSync(Daemon): if 'etag' in headers: headers['etag'] = headers['etag'].strip('"') headers['x-timestamp'] = row['created_at'] - headers['x-container-sync-key'] = sync_key + if realm and realm_key: + nonce = uuid.uuid4().hex + path = urlparse(sync_to).path + '/' + quote(row['name']) + sig = self.realms_conf.get_sig( + 'PUT', path, headers['x-timestamp'], nonce, realm_key, + user_key) + headers['x-container-sync-auth'] = '%s %s %s' % ( + realm, nonce, sig) + else: + headers['x-container-sync-key'] = user_key put_object(sync_to, name=row['name'], headers=headers, contents=FileLikeIter(body), proxy=self.proxy) diff --git a/test/unit/common/middleware/test_container_sync.py b/test/unit/common/middleware/test_container_sync.py new file mode 100644 index 0000000000..2956ccee2d --- /dev/null +++ b/test/unit/common/middleware/test_container_sync.py @@ -0,0 +1,233 @@ +# Copyright (c) 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import shutil +import tempfile +import unittest +import uuid + +from swift.common import swob +from swift.common.middleware import container_sync +from swift.proxy.controllers.base import _get_cache_key +from swift.proxy.controllers.info import InfoController + + +class FakeApp(object): + + def __call__(self, env, start_response): + if env.get('PATH_INFO') == '/info': + controller = InfoController( + app=None, version=None, expose_info=True, + disallowed_sections=[], admin_key=None) + handler = getattr(controller, env.get('REQUEST_METHOD')) + return handler(swob.Request(env))(env, start_response) + if env.get('swift.authorize_override'): + body = 'Response to Authorized Request' + else: + body = 'Pass-Through Response' + start_response('200 OK', [('Content-Length', str(len(body)))]) + return body + + +class TestContainerSync(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + with open( + os.path.join(self.tempdir, 'container-sync-realms.conf'), + 'w') as fp: + fp.write(''' +[US] +key = 9ff3b71c849749dbaec4ccdd3cbab62b +key2 = 1a0a5a0cbd66448084089304442d6776 +cluster_dfw1 = http://dfw1.host/v1/ + ''') + self.app = FakeApp() + self.conf = {'swift_dir': self.tempdir} + self.sync = container_sync.ContainerSync(self.app, self.conf) + + def tearDown(self): + shutil.rmtree(self.tempdir, ignore_errors=1) + + def test_pass_through(self): + req = swob.Request.blank('/v1/a/c') + resp = req.get_response(self.sync) + self.assertEqual(resp.status, '200 OK') + self.assertEqual(resp.body, 'Pass-Through Response') + + def test_not_enough_args(self): + req = swob.Request.blank( + '/v1/a/c', headers={'x-container-sync-auth': 'a'}) + resp = req.get_response(self.sync) + self.assertEqual(resp.status, '401 Unauthorized') + self.assertEqual( + resp.body, + 'X-Container-Sync-Auth header not valid; contact cluster operator ' + 'for support.') + self.assertTrue( + 'cs:not-3-args' in req.environ.get('swift.log_info'), + req.environ.get('swift.log_info')) + + def test_realm_miss(self): + req = swob.Request.blank( + '/v1/a/c', headers={'x-container-sync-auth': 'invalid nonce sig'}) + resp = req.get_response(self.sync) + self.assertEqual(resp.status, '401 Unauthorized') + self.assertEqual( + resp.body, + 'X-Container-Sync-Auth header not valid; contact cluster operator ' + 'for support.') + self.assertTrue( + 'cs:no-local-realm-key' in req.environ.get('swift.log_info'), + req.environ.get('swift.log_info')) + + def test_user_key_miss(self): + req = swob.Request.blank( + '/v1/a/c', headers={'x-container-sync-auth': 'US nonce sig'}) + resp = req.get_response(self.sync) + self.assertEqual(resp.status, '401 Unauthorized') + self.assertEqual( + resp.body, + 'X-Container-Sync-Auth header not valid; contact cluster operator ' + 'for support.') + self.assertTrue( + 'cs:no-local-user-key' in req.environ.get('swift.log_info'), + req.environ.get('swift.log_info')) + + def test_invalid_sig(self): + req = swob.Request.blank( + '/v1/a/c', headers={'x-container-sync-auth': 'US nonce sig'}) + req.environ[_get_cache_key('a', 'c')[1]] = {'sync_key': 'abc'} + resp = req.get_response(self.sync) + self.assertEqual(resp.status, '401 Unauthorized') + self.assertEqual( + resp.body, + 'X-Container-Sync-Auth header not valid; contact cluster operator ' + 'for support.') + self.assertTrue( + 'cs:invalid-sig' in req.environ.get('swift.log_info'), + req.environ.get('swift.log_info')) + + def test_valid_sig(self): + sig = self.sync.realms_conf.get_sig( + 'GET', '/v1/a/c', '0', 'nonce', + self.sync.realms_conf.key('US'), 'abc') + req = swob.Request.blank( + '/v1/a/c', headers={'x-container-sync-auth': 'US nonce ' + sig}) + req.environ[_get_cache_key('a', 'c')[1]] = {'sync_key': 'abc'} + resp = req.get_response(self.sync) + self.assertEqual(resp.status, '200 OK') + self.assertEqual(resp.body, 'Response to Authorized Request') + self.assertTrue( + 'cs:valid' in req.environ.get('swift.log_info'), + req.environ.get('swift.log_info')) + + def test_valid_sig2(self): + sig = self.sync.realms_conf.get_sig( + 'GET', '/v1/a/c', '0', 'nonce', + self.sync.realms_conf.key2('US'), 'abc') + req = swob.Request.blank( + '/v1/a/c', headers={'x-container-sync-auth': 'US nonce ' + sig}) + req.environ[_get_cache_key('a', 'c')[1]] = {'sync_key': 'abc'} + resp = req.get_response(self.sync) + self.assertEqual(resp.status, '200 OK') + self.assertEqual(resp.body, 'Response to Authorized Request') + self.assertTrue( + 'cs:valid' in req.environ.get('swift.log_info'), + req.environ.get('swift.log_info')) + + def test_info(self): + req = swob.Request.blank('/info') + resp = req.get_response(self.sync) + self.assertEqual(resp.status, '200 OK') + result = json.loads(resp.body) + self.assertEqual( + result.get('container_sync'), + {'realms': {'US': {'clusters': {'DFW1': {}}}}}) + + def test_info_always_fresh(self): + req = swob.Request.blank('/info') + resp = req.get_response(self.sync) + self.assertEqual(resp.status, '200 OK') + result = json.loads(resp.body) + self.assertEqual( + result.get('container_sync'), + {'realms': {'US': {'clusters': {'DFW1': {}}}}}) + with open( + os.path.join(self.tempdir, 'container-sync-realms.conf'), + 'w') as fp: + fp.write(''' +[US] +key = 9ff3b71c849749dbaec4ccdd3cbab62b +key2 = 1a0a5a0cbd66448084089304442d6776 +cluster_dfw1 = http://dfw1.host/v1/ + +[UK] +key = 400b3b357a80413f9d956badff1d9dfe +cluster_lon3 = http://lon3.host/v1/ + ''') + self.sync.realms_conf.reload() + req = swob.Request.blank('/info') + resp = req.get_response(self.sync) + self.assertEqual(resp.status, '200 OK') + result = json.loads(resp.body) + self.assertEqual( + result.get('container_sync'), + {'realms': { + 'US': {'clusters': {'DFW1': {}}}, + 'UK': {'clusters': {'LON3': {}}}}}) + + def test_allow_full_urls_setting(self): + req = swob.Request.blank( + '/v1/a/c', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'x-container-sync-to': 'http://host/v1/a/c'}) + resp = req.get_response(self.sync) + self.assertEqual(resp.status, '200 OK') + self.conf = {'swift_dir': self.tempdir, 'allow_full_urls': 'false'} + self.sync = container_sync.ContainerSync(self.app, self.conf) + req = swob.Request.blank( + '/v1/a/c', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'x-container-sync-to': 'http://host/v1/a/c'}) + resp = req.get_response(self.sync) + self.assertEqual(resp.status, '400 Bad Request') + self.assertEqual( + resp.body, + 'Full URLs are not allowed for X-Container-Sync-To values. Only ' + 'realm values of the format //realm/cluster/account/container are ' + 'allowed.\n') + + def test_filter(self): + app = FakeApp() + unique = uuid.uuid4().hex + sync = container_sync.filter_factory( + {'global': 'global_value', 'swift_dir': unique}, + **{'local': 'local_value'})(app) + self.assertEqual(sync.app, app) + self.assertEqual(sync.conf, { + 'global': 'global_value', 'swift_dir': unique, + 'local': 'local_value'}) + req = swob.Request.blank('/info') + resp = req.get_response(sync) + self.assertEqual(resp.status, '200 OK') + result = json.loads(resp.body) + self.assertEqual(result.get('container_sync'), {'realms': {}}) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_container_sync_realms.py b/test/unit/common/test_container_sync_realms.py new file mode 100644 index 0000000000..cc300e780d --- /dev/null +++ b/test/unit/common/test_container_sync_realms.py @@ -0,0 +1,187 @@ +# Copyright (c) 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest +import uuid + +from swift.common.container_sync_realms import ContainerSyncRealms +from test.unit import FakeLogger, temptree + + +class TestUtils(unittest.TestCase): + + def test_no_file_there(self): + unique = uuid.uuid4().hex + logger = FakeLogger() + csr = ContainerSyncRealms(unique, logger) + self.assertEqual( + logger.lines_dict, + {'debug': [ + "Could not load '%s': [Errno 2] No such file or directory: " + "'%s'" % (unique, unique)]}) + self.assertEqual(csr.mtime_check_interval, 300) + self.assertEqual(csr.realms(), []) + + def test_os_error(self): + fname = 'container-sync-realms.conf' + fcontents = '' + with temptree([fname], [fcontents]) as tempdir: + logger = FakeLogger() + fpath = os.path.join(tempdir, fname) + os.chmod(tempdir, 0) + csr = ContainerSyncRealms(fpath, logger) + try: + self.assertEqual( + logger.lines_dict, + {'error': [ + "Could not load '%s': [Errno 13] Permission denied: " + "'%s'" % (fpath, fpath)]}) + self.assertEqual(csr.mtime_check_interval, 300) + self.assertEqual(csr.realms(), []) + finally: + os.chmod(tempdir, 0700) + + def test_empty(self): + fname = 'container-sync-realms.conf' + fcontents = '' + with temptree([fname], [fcontents]) as tempdir: + logger = FakeLogger() + fpath = os.path.join(tempdir, fname) + csr = ContainerSyncRealms(fpath, logger) + self.assertEqual(logger.lines_dict, {}) + self.assertEqual(csr.mtime_check_interval, 300) + self.assertEqual(csr.realms(), []) + + def test_error_parsing(self): + fname = 'container-sync-realms.conf' + fcontents = 'invalid' + with temptree([fname], [fcontents]) as tempdir: + logger = FakeLogger() + fpath = os.path.join(tempdir, fname) + csr = ContainerSyncRealms(fpath, logger) + self.assertEqual( + logger.lines_dict, + {'error': [ + "Could not load '%s': File contains no section headers.\n" + "file: %s, line: 1\n" + "'invalid'" % (fpath, fpath)]}) + self.assertEqual(csr.mtime_check_interval, 300) + self.assertEqual(csr.realms(), []) + + def test_one_realm(self): + fname = 'container-sync-realms.conf' + fcontents = ''' +[US] +key = 9ff3b71c849749dbaec4ccdd3cbab62b +cluster_dfw1 = http://dfw1.host/v1/ +''' + with temptree([fname], [fcontents]) as tempdir: + logger = FakeLogger() + fpath = os.path.join(tempdir, fname) + csr = ContainerSyncRealms(fpath, logger) + self.assertEqual(logger.lines_dict, {}) + self.assertEqual(csr.mtime_check_interval, 300) + self.assertEqual(csr.realms(), ['US']) + self.assertEqual(csr.key('US'), '9ff3b71c849749dbaec4ccdd3cbab62b') + self.assertEqual(csr.key2('US'), None) + self.assertEqual(csr.clusters('US'), ['DFW1']) + self.assertEqual( + csr.endpoint('US', 'DFW1'), 'http://dfw1.host/v1/') + + def test_two_realms_and_change_a_default(self): + fname = 'container-sync-realms.conf' + fcontents = ''' +[DEFAULT] +mtime_check_interval = 60 + +[US] +key = 9ff3b71c849749dbaec4ccdd3cbab62b +cluster_dfw1 = http://dfw1.host/v1/ + +[UK] +key = e9569809dc8b4951accc1487aa788012 +key2 = f6351bd1cc36413baa43f7ba1b45e51d +cluster_lon3 = http://lon3.host/v1/ +''' + with temptree([fname], [fcontents]) as tempdir: + logger = FakeLogger() + fpath = os.path.join(tempdir, fname) + csr = ContainerSyncRealms(fpath, logger) + self.assertEqual(logger.lines_dict, {}) + self.assertEqual(csr.mtime_check_interval, 60) + self.assertEqual(sorted(csr.realms()), ['UK', 'US']) + self.assertEqual(csr.key('US'), '9ff3b71c849749dbaec4ccdd3cbab62b') + self.assertEqual(csr.key2('US'), None) + self.assertEqual(csr.clusters('US'), ['DFW1']) + self.assertEqual( + csr.endpoint('US', 'DFW1'), 'http://dfw1.host/v1/') + self.assertEqual(csr.key('UK'), 'e9569809dc8b4951accc1487aa788012') + self.assertEqual( + csr.key2('UK'), 'f6351bd1cc36413baa43f7ba1b45e51d') + self.assertEqual(csr.clusters('UK'), ['LON3']) + self.assertEqual( + csr.endpoint('UK', 'LON3'), 'http://lon3.host/v1/') + + def test_empty_realm(self): + fname = 'container-sync-realms.conf' + fcontents = ''' +[US] +''' + with temptree([fname], [fcontents]) as tempdir: + logger = FakeLogger() + fpath = os.path.join(tempdir, fname) + csr = ContainerSyncRealms(fpath, logger) + self.assertEqual(logger.lines_dict, {}) + self.assertEqual(csr.mtime_check_interval, 300) + self.assertEqual(csr.realms(), ['US']) + self.assertEqual(csr.key('US'), None) + self.assertEqual(csr.key2('US'), None) + self.assertEqual(csr.clusters('US'), []) + self.assertEqual(csr.endpoint('US', 'JUST_TESTING'), None) + + def test_bad_mtime_check_interval(self): + fname = 'container-sync-realms.conf' + fcontents = ''' +[DEFAULT] +mtime_check_interval = invalid +''' + with temptree([fname], [fcontents]) as tempdir: + logger = FakeLogger() + fpath = os.path.join(tempdir, fname) + csr = ContainerSyncRealms(fpath, logger) + self.assertEqual( + logger.lines_dict, + {'error': [ + "Error in '%s' with mtime_check_interval: invalid literal " + "for int() with base 10: 'invalid'" % fpath]}) + self.assertEqual(csr.mtime_check_interval, 300) + + def test_get_sig(self): + fname = 'container-sync-realms.conf' + fcontents = '' + with temptree([fname], [fcontents]) as tempdir: + logger = FakeLogger() + fpath = os.path.join(tempdir, fname) + csr = ContainerSyncRealms(fpath, logger) + self.assertEqual( + csr.get_sig( + 'GET', '/some/path', '1387212345.67890', 'my_nonce', + 'realm_key', 'user_key'), + '5a6eb486eb7b44ae1b1f014187a94529c3f9c8f9') + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 4a362b2295..6d66076c41 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -52,6 +52,7 @@ from swift.common.exceptions import (Timeout, MessageTimeout, ConnectionTimeout, LockTimeout, ReplicationLockTimeout) from swift.common import utils +from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.swob import Response from test.unit import FakeLogger @@ -1184,25 +1185,108 @@ log_name = %(yarr)s''' '1024Yi') def test_validate_sync_to(self): - for goodurl in ('http://1.1.1.1/v1/a/c/o', - 'http://1.1.1.1:8080/a/c/o', - 'http://2.2.2.2/a/c/o', - 'https://1.1.1.1/v1/a/c/o', - ''): - self.assertEquals(utils.validate_sync_to(goodurl, - ['1.1.1.1', '2.2.2.2']), - None) - for badurl in ('http://1.1.1.1', - 'httpq://1.1.1.1/v1/a/c/o', - 'http://1.1.1.1/v1/a/c/o?query', - 'http://1.1.1.1/v1/a/c/o#frag', - 'http://1.1.1.1/v1/a/c/o?query#frag', - 'http://1.1.1.1/v1/a/c/o?query=param', - 'http://1.1.1.1/v1/a/c/o?query=param#frag', - 'http://1.1.1.2/v1/a/c/o'): - self.assertNotEquals( - utils.validate_sync_to(badurl, ['1.1.1.1', '2.2.2.2']), - None) + fname = 'container-sync-realms.conf' + fcontents = ''' +[US] +key = 9ff3b71c849749dbaec4ccdd3cbab62b +cluster_dfw1 = http://dfw1.host/v1/ +''' + with temptree([fname], [fcontents]) as tempdir: + logger = FakeLogger() + fpath = os.path.join(tempdir, fname) + csr = ContainerSyncRealms(fpath, logger) + for realms_conf in (None, csr): + for goodurl, result in ( + ('http://1.1.1.1/v1/a/c', + (None, 'http://1.1.1.1/v1/a/c', None, None)), + ('http://1.1.1.1:8080/a/c', + (None, 'http://1.1.1.1:8080/a/c', None, None)), + ('http://2.2.2.2/a/c', + (None, 'http://2.2.2.2/a/c', None, None)), + ('https://1.1.1.1/v1/a/c', + (None, 'https://1.1.1.1/v1/a/c', None, None)), + ('//US/DFW1/a/c', + (None, 'http://dfw1.host/v1/a/c', 'US', + '9ff3b71c849749dbaec4ccdd3cbab62b')), + ('//us/DFW1/a/c', + (None, 'http://dfw1.host/v1/a/c', 'US', + '9ff3b71c849749dbaec4ccdd3cbab62b')), + ('//us/dfw1/a/c', + (None, 'http://dfw1.host/v1/a/c', 'US', + '9ff3b71c849749dbaec4ccdd3cbab62b')), + ('//', + (None, None, None, None)), + ('', + (None, None, None, None))): + if goodurl.startswith('//') and not realms_conf: + self.assertEquals( + utils.validate_sync_to( + goodurl, ['1.1.1.1', '2.2.2.2'], realms_conf), + (None, None, None, None)) + else: + self.assertEquals( + utils.validate_sync_to( + goodurl, ['1.1.1.1', '2.2.2.2'], realms_conf), + result) + for badurl, result in ( + ('http://1.1.1.1', + ('Path required in X-Container-Sync-To', None, None, + None)), + ('httpq://1.1.1.1/v1/a/c', + ('Invalid scheme \'httpq\' in X-Container-Sync-To, ' + 'must be "//", "http", or "https".', None, None, + None)), + ('http://1.1.1.1/v1/a/c?query', + ('Params, queries, and fragments not allowed in ' + 'X-Container-Sync-To', None, None, None)), + ('http://1.1.1.1/v1/a/c#frag', + ('Params, queries, and fragments not allowed in ' + 'X-Container-Sync-To', None, None, None)), + ('http://1.1.1.1/v1/a/c?query#frag', + ('Params, queries, and fragments not allowed in ' + 'X-Container-Sync-To', None, None, None)), + ('http://1.1.1.1/v1/a/c?query=param', + ('Params, queries, and fragments not allowed in ' + 'X-Container-Sync-To', None, None, None)), + ('http://1.1.1.1/v1/a/c?query=param#frag', + ('Params, queries, and fragments not allowed in ' + 'X-Container-Sync-To', None, None, None)), + ('http://1.1.1.2/v1/a/c', + ("Invalid host '1.1.1.2' in X-Container-Sync-To", + None, None, None)), + ('//us/invalid/a/c', + ("No cluster endpoint for 'us' 'invalid'", None, + None, None)), + ('//invalid/dfw1/a/c', + ("No realm key for 'invalid'", None, None, None)), + ('//us/invalid1/a/', + ("Invalid X-Container-Sync-To format " + "'//us/invalid1/a/'", None, None, None)), + ('//us/invalid1/a', + ("Invalid X-Container-Sync-To format " + "'//us/invalid1/a'", None, None, None)), + ('//us/invalid1/', + ("Invalid X-Container-Sync-To format " + "'//us/invalid1/'", None, None, None)), + ('//us/invalid1', + ("Invalid X-Container-Sync-To format " + "'//us/invalid1'", None, None, None)), + ('//us/', + ("Invalid X-Container-Sync-To format " + "'//us/'", None, None, None)), + ('//us', + ("Invalid X-Container-Sync-To format " + "'//us'", None, None, None))): + if badurl.startswith('//') and not realms_conf: + self.assertEquals( + utils.validate_sync_to( + badurl, ['1.1.1.1', '2.2.2.2'], realms_conf), + (None, None, None, None)) + else: + self.assertEquals( + utils.validate_sync_to( + badurl, ['1.1.1.1', '2.2.2.2'], realms_conf), + result) def test_TRUE_VALUES(self): for v in utils.TRUE_VALUES: diff --git a/test/unit/container/test_sync.py b/test/unit/container/test_sync.py index c8b75078a8..7c6b752557 100644 --- a/test/unit/container/test_sync.py +++ b/test/unit/container/test_sync.py @@ -626,15 +626,33 @@ class TestContainerSync(unittest.TestCase): sync.delete_object = orig_delete_object def test_container_sync_row_delete(self): + self._test_container_sync_row_delete(None, None) + + def test_container_sync_row_delete_using_realms(self): + self._test_container_sync_row_delete('US', 'realm_key') + + def _test_container_sync_row_delete(self, realm, realm_key): + orig_uuid = sync.uuid orig_delete_object = sync.delete_object try: + class FakeUUID(object): + class uuid4(object): + hex = 'abcdef' + + sync.uuid = FakeUUID def fake_delete_object(path, name=None, headers=None, proxy=None): self.assertEquals(path, 'http://sync/to/path') self.assertEquals(name, 'object') - self.assertEquals( - headers, - {'x-container-sync-key': 'key', 'x-timestamp': '1.2'}) + if realm: + self.assertEquals(headers, { + 'x-container-sync-auth': + 'US abcdef 90e95aabb45a6cdc0892a3db5535e7f918428c90', + 'x-timestamp': '1.2'}) + else: + self.assertEquals( + headers, + {'x-container-sync-key': 'key', 'x-timestamp': '1.2'}) self.assertEquals(proxy, 'http://proxy') sync.delete_object = fake_delete_object @@ -646,7 +664,8 @@ class TestContainerSync(unittest.TestCase): {'deleted': True, 'name': 'object', 'created_at': '1.2'}, 'http://sync/to/path', - 'key', FakeContainerBroker('broker'), 'info')) + 'key', FakeContainerBroker('broker'), 'info', realm, + realm_key)) self.assertEquals(cs.container_deletes, 1) exc = [] @@ -661,7 +680,8 @@ class TestContainerSync(unittest.TestCase): {'deleted': True, 'name': 'object', 'created_at': '1.2'}, 'http://sync/to/path', - 'key', FakeContainerBroker('broker'), 'info')) + 'key', FakeContainerBroker('broker'), 'info', realm, + realm_key)) self.assertEquals(cs.container_deletes, 1) self.assertEquals(len(exc), 1) self.assertEquals(str(exc[-1]), 'test exception') @@ -676,7 +696,8 @@ class TestContainerSync(unittest.TestCase): {'deleted': True, 'name': 'object', 'created_at': '1.2'}, 'http://sync/to/path', - 'key', FakeContainerBroker('broker'), 'info')) + 'key', FakeContainerBroker('broker'), 'info', realm, + realm_key)) self.assertEquals(cs.container_deletes, 1) self.assertEquals(len(exc), 2) self.assertEquals(str(exc[-1]), 'test client exception') @@ -692,29 +713,51 @@ class TestContainerSync(unittest.TestCase): {'deleted': True, 'name': 'object', 'created_at': '1.2'}, 'http://sync/to/path', - 'key', FakeContainerBroker('broker'), 'info')) + 'key', FakeContainerBroker('broker'), 'info', realm, + realm_key)) self.assertEquals(cs.container_deletes, 2) self.assertEquals(len(exc), 3) self.assertEquals(str(exc[-1]), 'test client exception: 404') finally: + sync.uuid = orig_uuid sync.delete_object = orig_delete_object def test_container_sync_row_put(self): + self._test_container_sync_row_put(None, None) + + def test_container_sync_row_put_using_realms(self): + self._test_container_sync_row_put('US', 'realm_key') + + def _test_container_sync_row_put(self, realm, realm_key): + orig_uuid = sync.uuid orig_shuffle = sync.shuffle orig_put_object = sync.put_object orig_direct_get_object = sync.direct_get_object try: + class FakeUUID(object): + class uuid4(object): + hex = 'abcdef' + + sync.uuid = FakeUUID sync.shuffle = lambda x: x def fake_put_object(sync_to, name=None, headers=None, contents=None, proxy=None): self.assertEquals(sync_to, 'http://sync/to/path') self.assertEquals(name, 'object') - self.assertEquals(headers, { - 'x-container-sync-key': 'key', - 'x-timestamp': '1.2', - 'other-header': 'other header value', - 'etag': 'etagvalue'}) + if realm: + self.assertEqual(headers, { + 'x-container-sync-auth': + 'US abcdef ef62c64bb88a33fa00722daa23d5d43253164962', + 'x-timestamp': '1.2', + 'etag': 'etagvalue', + 'other-header': 'other header value'}) + else: + self.assertEquals(headers, { + 'x-container-sync-key': 'key', + 'x-timestamp': '1.2', + 'other-header': 'other header value', + 'etag': 'etagvalue'}) self.assertEquals(contents.read(), 'contents') self.assertEquals(proxy, 'http://proxy') @@ -738,7 +781,7 @@ class TestContainerSync(unittest.TestCase): 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), { 'account': 'a', - 'container': 'c'})) + 'container': 'c'}, realm, realm_key)) self.assertEquals(cs.container_puts, 1) def fake_direct_get_object(node, part, account, container, obj, @@ -760,7 +803,7 @@ class TestContainerSync(unittest.TestCase): 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), { 'account': 'a', - 'container': 'c'})) + 'container': 'c'}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) exc = [] @@ -778,7 +821,7 @@ class TestContainerSync(unittest.TestCase): 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), { 'account': 'a', - 'container': 'c'})) + 'container': 'c'}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) self.assertEquals(len(exc), 3) self.assertEquals(str(exc[-1]), 'test exception') @@ -798,7 +841,7 @@ class TestContainerSync(unittest.TestCase): 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), { 'account': 'a', - 'container': 'c'})) + 'container': 'c'}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) self.assertEquals(len(exc), 3) self.assertEquals(str(exc[-1]), 'test client exception') @@ -823,7 +866,7 @@ class TestContainerSync(unittest.TestCase): 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), { 'account': 'a', - 'container': 'c'})) + 'container': 'c'}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) self.assert_(re.match('Unauth ', cs.logger.log_dict['info'][0][0][0])) @@ -841,7 +884,7 @@ class TestContainerSync(unittest.TestCase): 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), { 'account': 'a', - 'container': 'c'})) + 'container': 'c'}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) self.assert_(re.match('Not found ', cs.logger.log_dict['info'][0][0][0])) @@ -858,12 +901,13 @@ class TestContainerSync(unittest.TestCase): 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), { 'account': 'a', - 'container': 'c'})) + 'container': 'c'}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) self.assertTrue( cs.logger.log_dict['exception'][0][0][0].startswith( 'ERROR Syncing ')) finally: + sync.uuid = orig_uuid sync.shuffle = orig_shuffle sync.put_object = orig_put_object sync.direct_get_object = orig_direct_get_object