Merge "Deprecate per-service auto_create_account_prefix"

doc/manpages/account-server.conf.5

@@ -179,8 +179,8 @@ Logging level. The default is INFO.
 Enables request logging. The default is True.
 .IP "\fBset log_address\fR
 Logging address. The default is /dev/log.
-.IP "\fBauto_create_account_prefix\fR
+.IP "\fBauto_create_account_prefix [deprecated]\fR"
-The default is ".".
+The default is ".". Should be configured in swift.conf instead.
 .IP "\fBreplication_server\fR
 Configure parameter for creating specific server.
 To handle all verbs, including replication verbs, do not specify

doc/manpages/container-server.conf.5

@@ -191,8 +191,8 @@ Request timeout to external services. The default is 3 seconds.
 Connection timeout to external services. The default is 0.5 seconds.
 .IP \fBallow_versions\fR
 The default is false.
-.IP \fBauto_create_account_prefix\fR
+.IP "\fBauto_create_account_prefix [deprecated]\fR"
-The default is '.'.
+The default is '.'. Should be configured in swift.conf instead.
 .IP \fBreplication_server\fR
 Configure parameter for creating specific server.
 To handle all verbs, including replication verbs, do not specify
@@ -362,7 +362,7 @@ Request timeout to external services. The default is 3 seconds.
 Connection timeout to external services. The default is 0.5 seconds.
 .IP \fBcontainers_per_second\fR
 Maximum containers updated per second. Should be tuned according to individual system specs. 0 is unlimited. The default is 50.
-.IP \fBslowdown\fR
+.IP "\fBslowdown [deprecated]\fR"
 Slowdown will sleep that amount between containers. The default is 0.01 seconds. Deprecated in favor of containers_per_second
 .IP \fBaccount_suppression_time\fR
 Seconds to suppress updating an account that has generated an error. The default is 60 seconds.

doc/manpages/object-expirer.conf.5

@@ -202,8 +202,8 @@ This is normally \fBegg:swift#proxy_logging\fR. See proxy-server.conf-sample for
 .RS 3
 .IP \fBinterval\fR
 Replaces run_pause with the more standard "interval", which means the replicator won't pause unless it takes less than the interval set. The default is 300.
-.IP "\fBauto_create_account_prefix\fR
+.IP "\fBauto_create_account_prefix [deprecated]\fR"
-The default is ".".
+The default is ".". Should be configured in swift.conf instead.
 .IP \fBexpiring_objects_account_name\fR
 The default is 'expiring_objects'.
 .IP \fBreport_interval\fR

doc/manpages/object-server.conf.5

@@ -216,8 +216,8 @@ On PUTs, sync data every n MB. The default is 512.
 Comma separated list of headers that can be set in metadata on an object.
 This list is in addition to X-Object-Meta-* headers and cannot include Content-Type, etag, Content-Length, or deleted.
 The default is 'Content-Disposition, Content-Encoding, X-Delete-At, X-Object-Manifest, X-Static-Large-Object, Cache-Control, Content-Language, Expires, X-Robots-Tag'.
-.IP "\fBauto_create_account_prefix\fR"
+.IP "\fBauto_create_account_prefix [deprecated]\fR"
-The default is '.'.
+The default is '.'. Should be configured in swift.conf instead.
 .IP "\fBreplication_server\fR"
 Configure parameter for creating specific server
 To handle all verbs, including replication verbs, do not specify
@@ -504,7 +504,7 @@ Number of updater workers to spawn. The default is 1.
 Request timeout to external services. The default is 10 seconds.
 .IP \fBobjects_per_second\fR
 Maximum objects updated per second. Should be tuned according to individual system specs. 0 is unlimited. The default is 50.
-.IP \fBslowdown\fR
+.IP "\fBslowdown [deprecated]\fR"
 Slowdown will sleep that amount between objects. The default is 0.01 seconds. Deprecated in favor of objects_per_second.
 .IP "\fBrecon_cache_path\fR"
 The recon_cache_path simply sets the directory where stats for a few items will be stored.

doc/manpages/proxy-server.conf.5

@@ -1050,8 +1050,9 @@ is false.
 .IP \fBaccount_autocreate\fR
 If set to 'true' authorized accounts that do not yet exist within the Swift cluster
 will be automatically created. The default is set to false.
-.IP \fBauto_create_account_prefix\fR
+.IP "\fBauto_create_account_prefix [deprecated]\fR"
-Prefix used when automatically creating accounts. The default is '.'.
+Prefix used when automatically creating accounts. The default is '.'. Should
+be configured in swift.conf instead.
 .IP \fBmax_containers_per_account\fR
 If set to a positive value, trying to create a container when the account
 already has at least this maximum containers will result in a 403 Forbidden.

doc/manpages/swift.conf.5

@@ -201,6 +201,10 @@ Use a comma-separated list in case of multiple allowed versions, for example
 valid_api_versions = v0,v1,v2.
 This is only enforced for account, container and object requests. The allowed
 api versions are by default excluded from /info.
+.IP "\fBauto_create_account_prefix\fR"
+auto_create_account_prefix specifies the prefix for system accounts, such as
+those used by the object-expirer, and container-sharder.
+Default is ".".
 
 
 

doc/saio/swift/object-expirer.conf

@@ -25,7 +25,6 @@ log_level = INFO
 
 [object-expirer]
 interval = 300
-# auto_create_account_prefix = .
 # report_interval = 300
 # concurrency is the level of concurrency to use to do the work, this value
 # must be set to at least 1

doc/source/deployment_guide.rst

@@ -383,9 +383,9 @@ An example of common configuration file can be found at etc/swift.conf-sample
 
 The following configuration options are available:
 
-===================  ==========  =============================================
+==========================  ==========  =============================================
 Option                      Default     Description
--------------------  ----------  ---------------------------------------------
+--------------------------  ----------  ---------------------------------------------
 max_header_size             8192        max_header_size is the max number of bytes in
                                         the utf8 encoding of each header. Using 8192
                                         as default because eventlet use 8192 as max
@@ -402,7 +402,9 @@ extra_header_count   0           By default the maximum number of allowed
                                         If for some reason this is not enough (custom
                                         middleware for example) it can be increased
                                         with the extra_header_count constraint.
-===================  ==========  =============================================
+auto_create_account_prefix  .           Prefix used when automatically creating
+                                        accounts.
+==========================  ==========  =============================================
 
 ---------------------------
 Object Server Configuration
@@ -600,8 +602,6 @@ allowed_headers                    Content-Disposition,   Comma separated list o
                                    Content-Language,
                                    Expires,
                                    X-Robots-Tag
-auto_create_account_prefix         .                      Prefix used when automatically
-                                                          creating accounts.
 replication_server                                        Configure parameter for creating
                                                           specific server. To handle all verbs,
                                                           including replication verbs, do not
@@ -1017,8 +1017,6 @@ log_address                   /dev/log                        Logging directory
 interval                      300                             Time in seconds to wait between
                                                               expirer passes
 report_interval               300                             Frequency of status logs in seconds.
-auto_create_account_prefix    .                               Prefix used when automatically
-                                                              creating accounts.
 concurrency                   1                               Level of concurrency to use to do the work,
                                                               this value must be set to at least 1
 expiring_objects_account_name expiring_objects                name for legacy expirer task queue
@@ -1196,7 +1194,6 @@ set log_address                 /dev/log          Logging directory
 node_timeout                    3                 Request timeout to external services
 conn_timeout                    0.5               Connection timeout to external services
 allow_versions                  false             Enable/Disable object versioning feature
-auto_create_account_prefix      .                 Prefix used when automatically
 replication_server                                Configure parameter for creating
                                                   specific server. To handle all verbs,
                                                   including replication verbs, do not
@@ -1551,8 +1548,6 @@ set log_level                  INFO            Logging level
 set log_requests               True            Whether or not to log each
                                                request
 set log_address                /dev/log        Logging directory
-auto_create_account_prefix     .               Prefix used when automatically
-                                               creating accounts.
 replication_server                             Configure parameter for creating
                                                specific server. To handle all verbs,
                                                including replication verbs, do not

etc/account-server.conf-sample

@@ -91,8 +91,6 @@ use = egg:swift#account
 # set log_requests = true
 # set log_address = /dev/log
 #
-# auto_create_account_prefix = .
-#
 # Configure parameter for creating specific server
 # To handle all verbs, including replication verbs, do not specify
 # "replication_server" (this is the default). To only handle replication,

etc/container-server.conf-sample

@@ -84,10 +84,6 @@ bind_port = 6201
 # Work only with ionice_class.
 # ionice_class =
 # ionice_priority =
-#
-# The prefix used for hidden auto-created accounts, for example accounts in
-# which shard containers are created. Defaults to '.'.
-# auto_create_account_prefix = .
 
 [pipeline:main]
 pipeline = healthcheck recon container-server
@@ -104,7 +100,6 @@ use = egg:swift#container
 # node_timeout = 3
 # conn_timeout = 0.5
 # allow_versions = false
-# auto_create_account_prefix = .
 #
 # Configure parameter for creating specific server
 # To handle all verbs, including replication verbs, do not specify

etc/object-expirer.conf-sample

@@ -39,7 +39,6 @@
 
 [object-expirer]
 # interval = 300
-# auto_create_account_prefix = .
 # expiring_objects_account_name = expiring_objects
 # report_interval = 300
 #

etc/object-server.conf-sample

@@ -133,9 +133,6 @@ use = egg:swift#object
 # This list is in addition to X-Object-Meta-* headers and cannot include
 # Content-Type, etag, Content-Length, or deleted
 # allowed_headers = Content-Disposition, Content-Encoding, X-Delete-At, X-Object-Manifest, X-Static-Large-Object, Cache-Control, Content-Language, Expires, X-Robots-Tag
-#
-# auto_create_account_prefix = .
-#
 
 # The number of threads in eventlet's thread pool. Most IO will occur
 # in the object server's main thread, but certain "heavy" IO

etc/proxy-server.conf-sample

@@ -16,12 +16,13 @@ bind_port = 8080
 #
 # Allows the ability to withhold sections from showing up in the public calls
 # to /info.  You can withhold subsections by separating the dict level with a
-# ".".  The following would cause the sections 'container_quotas' and 'tempurl'
+# ".". Default value is 'swift.valid_api_versions, swift.auto_create_account_prefix'
-# to not be listed, and the key max_failed_deletes would be removed from
+# which allows all registered features to be listed via HTTP GET /info except
-# bulk_delete.  Default value is 'swift.valid_api_versions' which allows all
+# swift.valid_api_versions and swift.auto_create_account_prefix information.
-# registered features to be listed via HTTP GET /info except
+# As an example, the following would cause the sections 'container_quotas' and
-# swift.valid_api_versions information
+# 'tempurl' to not be listed, and the key max_failed_deletes would be removed from
-# disallowed_sections = swift.valid_api_versions, container_quotas, tempurl
+# bulk_delete.
+# disallowed_sections = swift.valid_api_versions, container_quotas, tempurl, bulk_delete.max_failed_deletes
 
 # Use an integer to override the number of pre-forked processes that will
 # accept connections.  Should default to the number of effective cpu
@@ -185,9 +186,6 @@ use = egg:swift#proxy
 # Comma separated list of Host headers to which the proxy will deny requests.
 # deny_host_headers =
 #
-# Prefix used when automatically creating accounts.
-# auto_create_account_prefix = .
-#
 # During GET and HEAD requests, storage nodes can be chosen at random
 # (shuffle), by using timing measurements (timing), or by using an explicit
 # region/zone match (affinity). Using timing measurements may allow for lower

etc/swift.conf-sample

@@ -194,3 +194,8 @@ aliases = yellow, orange
 # api versions are by default excluded from /info.
 
 # valid_api_versions = v1,v1.0
+
+# The prefix used for hidden auto-created accounts, for example accounts in
+# which shard containers are created. It defaults to '.'; don't change it.
+
+# auto_create_account_prefix = .

swift/account/server.py

@@ -32,7 +32,8 @@ from swift.common.utils import get_logger, hash_path, public, \
     Timestamp, storage_directory, config_true_value, \
     timing_stats, replication, get_log_line, \
     config_fallocate_value, fs_has_free_space
-from swift.common.constraints import valid_timestamp, check_utf8, check_drive
+from swift.common.constraints import valid_timestamp, check_utf8, \
+    check_drive, AUTO_CREATE_ACCOUNT_PREFIX
 from swift.common import constraints
 from swift.common.db_replicator import ReplicatorRpc
 from swift.common.base_storage_server import BaseStorageServer
@@ -85,8 +86,18 @@ class AccountController(BaseStorageServer):
         self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, AccountBroker,
                                             self.mount_check,
                                             logger=self.logger)
+        if conf.get('auto_create_account_prefix'):
+            self.logger.warning('Option auto_create_account_prefix is '
+                                'deprecated. Configure '
+                                'auto_create_account_prefix under the '
+                                'swift-constraints section of '
+                                'swift.conf. This option will '
+                                'be ignored in a future release.')
             self.auto_create_account_prefix = \
-            conf.get('auto_create_account_prefix') or '.'
+                conf['auto_create_account_prefix']
+        else:
+            self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
+
         swift.common.db.DB_PREALLOCATION = \
             config_true_value(conf.get('db_preallocation', 'f'))
         swift.common.db.QUERY_LOGGING = \

swift/common/constraints.py

@@ -39,6 +39,7 @@ MAX_ACCOUNT_NAME_LENGTH = 256
 MAX_CONTAINER_NAME_LENGTH = 256
 VALID_API_VERSIONS = ["v1", "v1.0"]
 EXTRA_HEADER_COUNT = 0
+AUTO_CREATE_ACCOUNT_PREFIX = '.'
 
 # If adding an entry to DEFAULT_CONSTRAINTS, note that
 # these constraints are automatically published by the
@@ -58,6 +59,7 @@ DEFAULT_CONSTRAINTS = {
     'max_container_name_length': MAX_CONTAINER_NAME_LENGTH,
     'valid_api_versions': VALID_API_VERSIONS,
     'extra_header_count': EXTRA_HEADER_COUNT,
+    'auto_create_account_prefix': AUTO_CREATE_ACCOUNT_PREFIX,
 }
 
 SWIFT_CONSTRAINTS_LOADED = False
@@ -76,7 +78,7 @@ def reload_constraints():
     constraints_conf = ConfigParser()
     if constraints_conf.read(utils.SWIFT_CONF_FILE):
         SWIFT_CONSTRAINTS_LOADED = True
-        for name in DEFAULT_CONSTRAINTS:
+        for name, default in DEFAULT_CONSTRAINTS.items():
             try:
                 value = constraints_conf.get('swift-constraints', name)
             except NoOptionError:
@@ -85,9 +87,12 @@ def reload_constraints():
                 # We are never going to find the section for another option
                 break
             else:
-                try:
+                if isinstance(default, int):
-                    value = int(value)
+                    value = int(value)  # Go ahead and let it error
-                except ValueError:
+                elif isinstance(default, str):
+                    pass  # No translation needed, I guess
+                else:
+                    # Hope we want a list!
                     value = utils.list_from_csv(value)
                 OVERRIDE_CONSTRAINTS[name] = value
     for name, default in DEFAULT_CONSTRAINTS.items():

swift/common/internal_client.py

@@ -25,6 +25,7 @@ import zlib
 from time import gmtime, strftime, time
 from zlib import compressobj
 
+from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX
 from swift.common.exceptions import ClientException
 from swift.common.http import (HTTP_NOT_FOUND, HTTP_MULTIPLE_CHOICES,
                                is_server_error)
@@ -158,7 +159,7 @@ class InternalClient(object):
     container_ring = pipeline_property('container_ring')
     account_ring = pipeline_property('account_ring')
     auto_create_account_prefix = pipeline_property(
-        'auto_create_account_prefix', default='.')
+        'auto_create_account_prefix', default=AUTO_CREATE_ACCOUNT_PREFIX)
 
     def make_request(
             self, method, path, headers, acceptable_statuses, body_file=None,

swift/common/utils.py

@@ -45,6 +45,7 @@ from random import random, shuffle
 from contextlib import contextmanager, closing
 import ctypes
 import ctypes.util
+from copy import deepcopy
 from optparse import OptionParser
 
 from tempfile import gettempdir, mkstemp, NamedTemporaryFile
@@ -328,7 +329,7 @@ def get_swift_info(admin=False, disallowed_sections=None):
     :returns: dictionary of information about the swift cluster.
     """
     disallowed_sections = disallowed_sections or []
-    info = dict(_swift_info)
+    info = deepcopy(_swift_info)
     for section in disallowed_sections:
         key_to_pop = None
         sub_section_dict = info

swift/common/wsgi.py

@@ -402,6 +402,9 @@ def loadapp(conf_file, global_conf=None, allow_modify_pipeline=True):
         func = getattr(app, 'modify_wsgi_pipeline', None)
         if func and allow_modify_pipeline:
             func(PipelineWrapper(ctx))
+        # cache the freshly created app so we con't have to redo
+        # initialization checks and log startup messages again
+        ctx.app_context.create = lambda: app
     return ctx.create()
 
 

swift/container/server.py

@@ -41,7 +41,8 @@ from swift.common.utils import get_logger, hash_path, public, \
     override_bytes_from_content_type, get_log_line, \
     config_fallocate_value, fs_has_free_space, list_from_csv, \
     ShardRange
-from swift.common.constraints import valid_timestamp, check_utf8, check_drive
+from swift.common.constraints import valid_timestamp, check_utf8, \
+    check_drive, AUTO_CREATE_ACCOUNT_PREFIX
 from swift.common import constraints
 from swift.common.bufferedhttp import http_connect
 from swift.common.exceptions import ConnectionTimeout
@@ -143,8 +144,17 @@ class ContainerController(BaseStorageServer):
         self.replicator_rpc = ContainerReplicatorRpc(
             self.root, DATADIR, ContainerBroker, self.mount_check,
             logger=self.logger)
+        if conf.get('auto_create_account_prefix'):
+            self.logger.warning('Option auto_create_account_prefix is '
+                                'deprecated. Configure '
+                                'auto_create_account_prefix under the '
+                                'swift-constraints section of '
+                                'swift.conf. This option will '
+                                'be ignored in a future release.')
             self.auto_create_account_prefix = \
-            conf.get('auto_create_account_prefix') or '.'
+                conf['auto_create_account_prefix']
+        else:
+            self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
         if config_true_value(conf.get('allow_versions', 'f')):
             self.save_headers.append('x-versions-location')
         if 'allow_versions' in conf:

swift/container/sharder.py

@@ -25,7 +25,7 @@ from six.moves.urllib.parse import quote
 from eventlet import Timeout
 
 from swift.common import internal_client
-from swift.common.constraints import check_drive
+from swift.common.constraints import check_drive, AUTO_CREATE_ACCOUNT_PREFIX
 from swift.common.direct_client import (direct_put_container,
                                         DirectClientException)
 from swift.common.exceptions import DeviceUnavailable
@@ -334,8 +334,18 @@ class ContainerSharder(ContainerReplicator):
     def __init__(self, conf, logger=None):
         logger = logger or get_logger(conf, log_route='container-sharder')
         super(ContainerSharder, self).__init__(conf, logger=logger)
-        self.shards_account_prefix = (
+        if conf.get('auto_create_account_prefix'):
-            (conf.get('auto_create_account_prefix') or '.') + 'shards_')
+            self.logger.warning('Option auto_create_account_prefix is '
+                                'deprecated. Configure '
+                                'auto_create_account_prefix under the '
+                                'swift-constraints section of '
+                                'swift.conf. This option will '
+                                'be ignored in a future release.')
+            auto_create_account_prefix = \
+                self.conf['auto_create_account_prefix']
+        else:
+            auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
+        self.shards_account_prefix = (auto_create_account_prefix + 'shards_')
 
         def percent_value(key, default):
             try:

swift/obj/expirer.py

@@ -25,6 +25,7 @@ import hashlib
 from eventlet import sleep, Timeout
 from eventlet.greenpool import GreenPool
 
+from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX
 from swift.common.daemon import Daemon
 from swift.common.internal_client import InternalClient, UnexpectedResponse
 from swift.common.utils import get_logger, dump_recon_cache, split_path, \
@@ -112,8 +113,19 @@ class ObjectExpirer(Daemon):
         self.reclaim_age = int(conf.get('reclaim_age', 604800))
 
     def read_conf_for_queue_access(self, swift):
-        self.expiring_objects_account = \
+        if self.conf.get('auto_create_account_prefix'):
-            (self.conf.get('auto_create_account_prefix') or '.') + \
+            self.logger.warning('Option auto_create_account_prefix is '
+                                'deprecated. Configure '
+                                'auto_create_account_prefix under the '
+                                'swift-constraints section of '
+                                'swift.conf. This option will '
+                                'be ignored in a future release.')
+            auto_create_account_prefix = \
+                self.conf['auto_create_account_prefix']
+        else:
+            auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
+
+        self.expiring_objects_account = auto_create_account_prefix + \
             (self.conf.get('expiring_objects_account_name') or
              'expiring_objects')
 

swift/obj/server.py

@@ -40,7 +40,7 @@ from swift.common.utils import public, get_logger, \
     normalize_timestamp
 from swift.common.bufferedhttp import http_connect
 from swift.common.constraints import check_object_creation, \
-    valid_timestamp, check_utf8
+    valid_timestamp, check_utf8, AUTO_CREATE_ACCOUNT_PREFIX
 from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \
     DiskFileNotExist, DiskFileCollision, DiskFileNoSpace, DiskFileDeleted, \
     DiskFileDeviceUnavailable, DiskFileExpired, ChunkReadTimeout, \
@@ -174,8 +174,18 @@ class ObjectController(BaseStorageServer):
         for header in extra_allowed_headers:
             if header not in RESERVED_DATAFILE_META:
                 self.allowed_headers.add(header)
+        if conf.get('auto_create_account_prefix'):
+            self.logger.warning('Option auto_create_account_prefix is '
+                                'deprecated. Configure '
+                                'auto_create_account_prefix under the '
+                                'swift-constraints section of '
+                                'swift.conf. This option will '
+                                'be ignored in a future release.')
             self.auto_create_account_prefix = \
-            conf.get('auto_create_account_prefix') or '.'
+                conf['auto_create_account_prefix']
+        else:
+            self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
+
         self.expiring_objects_account = self.auto_create_account_prefix + \
             (conf.get('expiring_objects_account_name') or 'expiring_objects')
         self.expiring_objects_container_divisor = \

swift/proxy/controllers/base.py

@@ -347,7 +347,8 @@ def get_container_info(env, app, swift_source=None):
         # account is successful whether the account actually has .db files
         # on disk or not.
         is_autocreate_account = account.startswith(
-            getattr(app, 'auto_create_account_prefix', '.'))
+            getattr(app, 'auto_create_account_prefix',
+                    constraints.AUTO_CREATE_ACCOUNT_PREFIX))
         if not is_autocreate_account:
             account_info = get_account_info(env, app, swift_source)
             if not account_info or not is_success(account_info['status']):

swift/proxy/server.py

@@ -223,8 +223,18 @@ class Application(object):
                        [os.path.join(swift_dir, 'mime.types')])
         self.account_autocreate = \
             config_true_value(conf.get('account_autocreate', 'no'))
-        self.auto_create_account_prefix = (
+        if conf.get('auto_create_account_prefix'):
-            conf.get('auto_create_account_prefix') or '.')
+            self.logger.warning('Option auto_create_account_prefix is '
+                                'deprecated. Configure '
+                                'auto_create_account_prefix under the '
+                                'swift-constraints section of '
+                                'swift.conf. This option will '
+                                'be ignored in a future release.')
+            self.auto_create_account_prefix = \
+                conf['auto_create_account_prefix']
+        else:
+            self.auto_create_account_prefix = \
+                constraints.AUTO_CREATE_ACCOUNT_PREFIX
         self.expiring_objects_account = self.auto_create_account_prefix + \
             (conf.get('expiring_objects_account_name') or 'expiring_objects')
         self.expiring_objects_container_divisor = \
@@ -295,7 +305,10 @@ class Application(object):
         self.expose_info = config_true_value(
             conf.get('expose_info', 'yes'))
         self.disallowed_sections = list_from_csv(
-            conf.get('disallowed_sections', 'swift.valid_api_versions'))
+            conf.get('disallowed_sections', ', '.join([
+                'swift.auto_create_account_prefix',
+                'swift.valid_api_versions',
+            ])))
         self.admin_key = conf.get('admin_key', None)
         register_swift_info(
             version=swift_version,

test/probe/test_container_merge_policy_index.py

@@ -23,6 +23,7 @@ from swift.common.internal_client import InternalClient
 from swift.common import utils, direct_client
 from swift.common.storage_policy import POLICIES
 from swift.common.http import HTTP_NOT_FOUND
+from swift.container.reconciler import MISPLACED_OBJECTS_ACCOUNT
 from test.probe.brain import BrainSplitter
 from test.probe.common import (ReplProbeTest, ENABLED_POLICIES,
                                POLICIES_BY_TYPE, REPL_POLICY)
@@ -495,8 +496,8 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
                 server.once(number=self.config_number(node))
 
         # verify entry in the queue for the "misplaced" new_policy
-        for container in int_client.iter_containers('.misplaced_objects'):
+        for container in int_client.iter_containers(MISPLACED_OBJECTS_ACCOUNT):
-            for obj in int_client.iter_objects('.misplaced_objects',
+            for obj in int_client.iter_objects(MISPLACED_OBJECTS_ACCOUNT,
                                                container['name']):
                 expected = '%d:/%s/%s/%s' % (new_policy, self.account,
                                              self.container_name,
@@ -519,8 +520,8 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
         self.get_to_final_state()
 
         # verify entry in the queue
-        for container in int_client.iter_containers('.misplaced_objects'):
+        for container in int_client.iter_containers(MISPLACED_OBJECTS_ACCOUNT):
-            for obj in int_client.iter_objects('.misplaced_objects',
+            for obj in int_client.iter_objects(MISPLACED_OBJECTS_ACCOUNT,
                                                container['name']):
                 expected = '%d:/%s/%s/%s' % (old_policy, self.account,
                                              self.container_name,
@@ -540,8 +541,8 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
 
         # make sure the queue is settled
         self.get_to_final_state()
-        for container in int_client.iter_containers('.misplaced_objects'):
+        for container in int_client.iter_containers(MISPLACED_OBJECTS_ACCOUNT):
-            for obj in int_client.iter_objects('.misplaced_objects',
+            for obj in int_client.iter_objects(MISPLACED_OBJECTS_ACCOUNT,
                                                container['name']):
                 self.fail('Found unexpected object %r in the queue' % obj)
 

test/unit/account/test_server.py

@@ -51,9 +51,10 @@ class TestAccountController(unittest.TestCase):
         self.testdir_base = mkdtemp()
         self.testdir = os.path.join(self.testdir_base, 'account_server')
         mkdirs(os.path.join(self.testdir, 'sda1'))
+        self.logger = debug_logger()
         self.controller = AccountController(
             {'devices': self.testdir, 'mount_check': 'false'},
-            logger=debug_logger())
+            logger=self.logger)
         self.ts = make_timestamp_iter()
 
     def tearDown(self):
@@ -64,6 +65,22 @@ class TestAccountController(unittest.TestCase):
             if err.errno != errno.ENOENT:
                 raise
 
+    def test_init(self):
+        conf = {
+            'devices': self.testdir,
+            'mount_check': 'false',
+        }
+        AccountController(conf, logger=self.logger)
+        self.assertEqual(self.logger.get_lines_for_level('warning'), [])
+        conf['auto_create_account_prefix'] = '-'
+        AccountController(conf, logger=self.logger)
+        self.assertEqual(self.logger.get_lines_for_level('warning'), [
+            'Option auto_create_account_prefix is deprecated. '
+            'Configure auto_create_account_prefix under the '
+            'swift-constraints section of swift.conf. This option '
+            'will be ignored in a future release.'
+        ])
+
     def test_OPTIONS(self):
         server_handler = AccountController(
             {'devices': self.testdir, 'mount_check': 'false'})

test/unit/common/test_constraints.py

@@ -649,10 +649,15 @@ class TestConstraintsConfig(unittest.TestCase):
                 f.flush()
                 with mock.patch.object(utils, 'SWIFT_CONF_FILE', f.name):
                     constraints.reload_constraints()
-            for key in constraints.DEFAULT_CONSTRAINTS:
+            for key, default in constraints.DEFAULT_CONSTRAINTS.items():
                 # module level attrs should all be 1
                 module_level_value = getattr(constraints, key.upper())
+                if isinstance(default, int):
                     self.assertEqual(module_level_value, 1)
+                elif isinstance(default, str):
+                    self.assertEqual(module_level_value, '1')
+                else:
+                    self.assertEqual(module_level_value, ['1'])
                 # all keys should be in OVERRIDE
                 self.assertEqual(constraints.OVERRIDE_CONSTRAINTS[key],
                                  module_level_value)

test/unit/common/test_internal_client.py

@@ -25,14 +25,13 @@ from textwrap import dedent
 import six
 from six.moves import range, zip_longest
 from six.moves.urllib.parse import quote, parse_qsl
-from test.unit import FakeLogger
 from swift.common import exceptions, internal_client, swob
 from swift.common.header_key_dict import HeaderKeyDict
 from swift.common.storage_policy import StoragePolicy
 from swift.common.middleware.proxy_logging import ProxyLoggingMiddleware
 
 from test.unit import with_tempdir, write_fake_ring, patch_policies, \
-    DebugLogger
+    debug_logger
 from test.unit.common.middleware.helpers import FakeSwift, LeakTrackingIter
 
 if six.PY3:
@@ -256,8 +255,17 @@ class TestInternalClient(unittest.TestCase):
         write_fake_ring(container_ring_path)
         object_ring_path = os.path.join(tempdir, 'object.ring.gz')
         write_fake_ring(object_ring_path)
+        logger = debug_logger('test-ic')
+        self.assertEqual(logger.get_lines_for_level('warning'), [])
         with patch_policies([StoragePolicy(0, 'legacy', True)]):
+            with mock.patch('swift.proxy.server.get_logger',
+                            lambda *a, **kw: logger):
                 client = internal_client.InternalClient(conf_path, 'test', 1)
+            self.assertEqual(logger.get_lines_for_level('warning'), [
+                'Option auto_create_account_prefix is deprecated. '
+                'Configure auto_create_account_prefix under the '
+                'swift-constraints section of swift.conf. This option will '
+                'be ignored in a future release.'])
             self.assertEqual(client.account_ring,
                              client.app.app.app.account_ring)
             self.assertEqual(client.account_ring.serialized_path,
@@ -454,7 +462,7 @@ class TestInternalClient(unittest.TestCase):
     def test_make_request_error_case(self):
         class InternalClient(internal_client.InternalClient):
             def __init__(self):
-                self.logger = DebugLogger()
+                self.logger = debug_logger('test-ic')
                 # wrap the fake app with ProxyLoggingMiddleware
                 self.app = ProxyLoggingMiddleware(
                     self.fake_app, {}, self.logger)
@@ -484,7 +492,7 @@ class TestInternalClient(unittest.TestCase):
     def test_make_request_acceptable_status_not_2xx(self):
         class InternalClient(internal_client.InternalClient):
             def __init__(self, resp_status):
-                self.logger = DebugLogger()
+                self.logger = debug_logger('test-ic')
                 # wrap the fake app with ProxyLoggingMiddleware
                 self.app = ProxyLoggingMiddleware(
                     self.fake_app, {}, self.logger)
@@ -1408,7 +1416,7 @@ class TestSimpleClient(unittest.TestCase):
             urlopen.return_value.getcode.return_value = 200
             urlopen.return_value.info.return_value = {'content-length': '345'}
             sc = internal_client.SimpleClient(url='http://127.0.0.1')
-            logger = FakeLogger()
+            logger = debug_logger('test-ic')
             retval = sc.retry_request(
                 method, headers={'content-length': '123'}, logger=logger)
             self.assertEqual(urlopen.call_count, 1)
@@ -1417,10 +1425,11 @@ class TestSimpleClient(unittest.TestCase):
                                        data=None)
             self.assertEqual([{'content-length': '345'}, None], retval)
             self.assertEqual(method, request.return_value.get_method())
-            self.assertEqual(logger.log_dict['debug'], [(
+            self.assertEqual(logger.get_lines_for_level('debug'), [
-                ('-> 2014-05-27T20:54:11 ' + method +
+                '-> 2014-05-27T20:54:11 ' + method +
                 ' http://127.0.0.1%3Fformat%3Djson 200 '
-                 '123 345 1401224050.98 1401224051.98 1.0 -',), {})])
+                '123 345 1401224050.98 1401224051.98 1.0 -'
+            ])
 
             # Check if JSON is decoded
             urlopen.return_value.read.return_value = b'{}'

test/unit/container/test_server.py

@@ -116,9 +116,18 @@ class TestContainerController(unittest.TestCase):
 
     def test_creation(self):
         # later config should be extended to assert more config options
-        replicator = container_server.ContainerController(
+        app = container_server.ContainerController(
-            {'node_timeout': '3.5'})
+            {'node_timeout': '3.5'}, logger=self.logger)
-        self.assertEqual(replicator.node_timeout, 3.5)
+        self.assertEqual(app.node_timeout, 3.5)
+        self.assertEqual(self.logger.get_lines_for_level('warning'), [])
+        app = container_server.ContainerController(
+            {'auto_create_account_prefix': '-'}, logger=self.logger)
+        self.assertEqual(self.logger.get_lines_for_level('warning'), [
+            'Option auto_create_account_prefix is deprecated. '
+            'Configure auto_create_account_prefix under the '
+            'swift-constraints section of swift.conf. This option '
+            'will be ignored in a future release.'
+        ])
 
     def test_get_and_validate_policy_index(self):
         # no policy is OK

test/unit/container/test_sharder.py

@@ -45,7 +45,7 @@ from swift.common.utils import ShardRange, Timestamp, hash_path, \
     encode_timestamps, parse_db_filename, quorum_size, Everything
 from test import annotate_failure
 
-from test.unit import FakeLogger, debug_logger, FakeRing, \
+from test.unit import debug_logger, FakeRing, \
     make_timestamp_iter, unlink_files, mocked_http_conn, mock_timestamp_now, \
     attach_fake_replication_rpc
 
@@ -54,6 +54,7 @@ class BaseTestSharder(unittest.TestCase):
     def setUp(self):
         self.tempdir = mkdtemp()
         self.ts_iter = make_timestamp_iter()
+        self.logger = debug_logger('sharder-test')
 
     def tearDown(self):
         shutil.rmtree(self.tempdir, ignore_errors=True)
@@ -74,7 +75,7 @@ class BaseTestSharder(unittest.TestCase):
         db_file = os.path.join(datadir, filename)
         broker = ContainerBroker(
             db_file, account=account, container=container,
-            logger=debug_logger())
+            logger=self.logger)
         broker.initialize()
         return broker
 
@@ -123,7 +124,9 @@ class BaseTestSharder(unittest.TestCase):
 
 class TestSharder(BaseTestSharder):
     def test_init(self):
-        def do_test(conf, expected):
+        def do_test(conf, expected, logger=self.logger):
+            if logger:
+                logger.clear()
             with mock.patch(
                     'swift.container.sharder.internal_client.InternalClient') \
                     as mock_ic:
@@ -131,17 +134,15 @@ class TestSharder(BaseTestSharder):
                         as mock_ring:
                     mock_ring.return_value = mock.MagicMock()
                     mock_ring.return_value.replica_count = 3
-                    sharder = ContainerSharder(conf)
+                    sharder = ContainerSharder(conf, logger=logger)
             mock_ring.assert_called_once_with(
                 '/etc/swift', ring_name='container')
-            self.assertEqual(
-                'container-sharder', sharder.logger.logger.name)
             for k, v in expected.items():
                 self.assertTrue(hasattr(sharder, k), 'Missing attr %s' % k)
                 self.assertEqual(v, getattr(sharder, k),
                                  'Incorrect value: expected %s=%s but got %s' %
                                  (k, v, getattr(sharder, k)))
-            return mock_ic
+            return sharder, mock_ic
 
         expected = {
             'mount_check': True, 'bind_ip': '0.0.0.0', 'port': 6201,
@@ -165,7 +166,9 @@ class TestSharder(BaseTestSharder):
             'shard_replication_quorum': 2,
             'existing_shard_replication_quorum': 2
         }
-        mock_ic = do_test({}, expected)
+        sharder, mock_ic = do_test({}, expected, logger=None)
+        self.assertEqual(
+            'container-sharder', sharder.logger.logger.name)
         mock_ic.assert_called_once_with(
             '/etc/swift/internal-client.conf', 'Swift Container Sharder', 3,
             allow_modify_pipeline=False)
@@ -215,16 +218,33 @@ class TestSharder(BaseTestSharder):
             'shard_replication_quorum': 1,
             'existing_shard_replication_quorum': 0
         }
-        mock_ic = do_test(conf, expected)
+        sharder, mock_ic = do_test(conf, expected)
         mock_ic.assert_called_once_with(
             '/etc/swift/my-sharder-ic.conf', 'Swift Container Sharder', 2,
             allow_modify_pipeline=False)
+        self.assertEqual(self.logger.get_lines_for_level('warning'), [
+            'Option auto_create_account_prefix is deprecated. '
+            'Configure auto_create_account_prefix under the '
+            'swift-constraints section of swift.conf. This option '
+            'will be ignored in a future release.'])
 
         expected.update({'shard_replication_quorum': 3,
                          'existing_shard_replication_quorum': 3})
         conf.update({'shard_replication_quorum': 4,
                      'existing_shard_replication_quorum': 4})
         do_test(conf, expected)
+        warnings = self.logger.get_lines_for_level('warning')
+        self.assertEqual(warnings[:1], [
+            'Option auto_create_account_prefix is deprecated. '
+            'Configure auto_create_account_prefix under the '
+            'swift-constraints section of swift.conf. This option '
+            'will be ignored in a future release.'])
+        self.assertEqual(warnings[1:], [
+            'shard_replication_quorum of 4 exceeds replica count 3, '
+            'reducing to 3',
+            'existing_shard_replication_quorum of 4 exceeds replica count 3, '
+            'reducing to 3',
+        ])
 
         with self.assertRaises(ValueError) as cm:
             do_test({'shard_shrink_point': 101}, {})
@@ -502,7 +522,6 @@ class TestSharder(BaseTestSharder):
                 mock.patch('swift.container.sharder.is_local_device',
                            return_value=True):
             sharder.reported = time.time()
-            sharder.logger = debug_logger()
             brokers = []
             device_ids = set(d['id'] for d in sharder.ring.devs)
 
@@ -709,6 +728,7 @@ class TestSharder(BaseTestSharder):
 
     @contextmanager
     def _mock_sharder(self, conf=None, replicas=3):
+        self.logger.clear()
         conf = conf or {}
         conf['devices'] = self.tempdir
         with mock.patch(
@@ -716,7 +736,7 @@ class TestSharder(BaseTestSharder):
             with mock.patch(
                     'swift.common.db_replicator.ring.Ring',
                     lambda *args, **kwargs: FakeRing(replicas=replicas)):
-                sharder = ContainerSharder(conf, logger=FakeLogger())
+                sharder = ContainerSharder(conf, logger=self.logger)
                 sharder._local_device_ids = {0, 1, 2}
                 sharder._replicate_object = mock.MagicMock(
                     return_value=(True, [True] * sharder.ring.replica_count))
@@ -1095,7 +1115,6 @@ class TestSharder(BaseTestSharder):
 
         # run cleave again - should process the fourth range
         with self._mock_sharder(conf=conf) as sharder:
-            sharder.logger = debug_logger()
             self.assertFalse(sharder._cleave(broker))
 
         expected = {'attempted': 1, 'success': 1, 'failure': 0,
@@ -1860,7 +1879,6 @@ class TestSharder(BaseTestSharder):
                     side_effect=[(True, [True, True, True]),  # misplaced obj
                                  (False, [False, True, True])])
                 sharder._audit_container = mock.MagicMock()
-                sharder.logger = debug_logger()
                 sharder._process_broker(broker, node, 99)
 
         self.assertEqual(SHARDED, broker.get_db_state())
@@ -2018,7 +2036,6 @@ class TestSharder(BaseTestSharder):
                 sharder._replicate_object = mock.MagicMock(
                     side_effect=[(False, [False, True, True])])
                 sharder._audit_container = mock.MagicMock()
-                sharder.logger = debug_logger()
                 sharder._process_broker(broker, node, 99)
         self.assertEqual(SHARDING, broker.get_db_state())
         self.assertEqual(ShardRange.SHARDING,
@@ -3423,7 +3440,6 @@ class TestSharder(BaseTestSharder):
         self.assertTrue(broker.set_sharded_state())
 
         with self._mock_sharder() as sharder:
-            sharder.logger = debug_logger()
             sharder._move_misplaced_objects(broker)
 
         sharder._replicate_object.assert_has_calls(
@@ -4034,7 +4050,6 @@ class TestSharder(BaseTestSharder):
                 with self._mock_sharder() as sharder:
                     with mock_timestamp_now() as now:
                         with mock.patch.object(sharder, '_audit_container'):
-                            sharder.logger = debug_logger()
                             sharder._process_broker(broker, node, 99)
                             own_shard_range = broker.get_own_shard_range(
                                 no_default=True)
@@ -4149,8 +4164,6 @@ class TestSharder(BaseTestSharder):
         own_sr.epoch = epoch
         broker.merge_shard_ranges([own_sr])
         with self._mock_sharder() as sharder:
-
-            sharder.logger = debug_logger()
             with mock_timestamp_now() as now:
                 # we're not testing rest of the process here so prevent any
                 # attempt to progress shard range states
@@ -4463,7 +4476,6 @@ class TestSharder(BaseTestSharder):
 
         def call_audit_container(exc=None):
             with self._mock_sharder() as sharder:
-                sharder.logger = debug_logger()
                 with mock.patch.object(sharder, '_audit_root_container') \
                         as mocked, mock.patch.object(
                             sharder, 'int_client') as mock_swift:

test/unit/obj/test_expirer.py

@@ -154,6 +154,20 @@ class TestObjectExpirer(TestCase):
         internal_client.sleep = self.old_sleep
         internal_client.loadapp = self.old_loadapp
 
+    def test_init(self):
+        x = expirer.ObjectExpirer({}, logger=self.logger)
+        self.assertEqual(self.logger.get_lines_for_level('warning'), [])
+        self.assertEqual(x.expiring_objects_account, '.expiring_objects')
+        x = expirer.ObjectExpirer({'auto_create_account_prefix': '-'},
+                                  logger=self.logger)
+        self.assertEqual(self.logger.get_lines_for_level('warning'), [
+            'Option auto_create_account_prefix is deprecated. '
+            'Configure auto_create_account_prefix under the '
+            'swift-constraints section of swift.conf. This option '
+            'will be ignored in a future release.'
+        ])
+        self.assertEqual(x.expiring_objects_account, '-expiring_objects')
+
     def test_get_process_values_from_kwargs(self):
         x = expirer.ObjectExpirer({})
         vals = {

test/unit/obj/test_server.py

@@ -148,8 +148,9 @@ class TestObjectController(unittest.TestCase):
         mkdirs(os.path.join(self.testdir, 'sda1'))
         self.conf = {'devices': self.testdir, 'mount_check': 'false',
                      'container_update_timeout': 0.0}
+        self.logger = debug_logger()
         self.object_controller = object_server.ObjectController(
-            self.conf, logger=debug_logger())
+            self.conf, logger=self.logger)
         self.object_controller.bytes_per_sync = 1
         self._orig_tpool_exc = tpool.execute
         tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs)
@@ -174,6 +175,27 @@ class TestObjectController(unittest.TestCase):
             self.policy = policy
             yield policy
 
+    def test_init(self):
+        conf = {
+            'devices': self.testdir,
+            'mount_check': 'false',
+            'container_update_timeout': 0.0,
+        }
+        app = object_server.ObjectController(conf, logger=self.logger)
+        self.assertEqual(app.container_update_timeout, 0.0)
+        self.assertEqual(app.auto_create_account_prefix, '.')
+        self.assertEqual(self.logger.get_lines_for_level('warning'), [])
+
+        conf['auto_create_account_prefix'] = '-'
+        app = object_server.ObjectController(conf, logger=self.logger)
+        self.assertEqual(app.auto_create_account_prefix, '-')
+        self.assertEqual(self.logger.get_lines_for_level('warning'), [
+            'Option auto_create_account_prefix is deprecated. '
+            'Configure auto_create_account_prefix under the '
+            'swift-constraints section of swift.conf. This option '
+            'will be ignored in a future release.'
+        ])
+
     def check_all_api_methods(self, obj_name='o', alt_res=None):
         path = '/sda1/p/a/c/%s' % obj_name
         body = b'SPECIAL_STRING'

test/unit/proxy/test_server.py

@@ -571,11 +571,31 @@ class TestController(unittest.TestCase):
 
 @patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())])
 class TestProxyServerConfiguration(unittest.TestCase):
+
+    def setUp(self):
+        self.logger = debug_logger('test-proxy-config')
+
     def _make_app(self, conf):
+        self.logger.clear()
         # helper function to instantiate a proxy server instance
         return proxy_server.Application(conf, FakeMemcache(),
                                         container_ring=FakeRing(),
-                                        account_ring=FakeRing())
+                                        account_ring=FakeRing(),
+                                        logger=self.logger)
+
+    def test_auto_create_account(self):
+        app = self._make_app({})
+        self.assertEqual(app.auto_create_account_prefix, '.')
+        self.assertEqual(self.logger.get_lines_for_level('warning'), [])
+
+        app = self._make_app({'auto_create_account_prefix': '-'})
+        self.assertEqual(app.auto_create_account_prefix, '-')
+        self.assertEqual(self.logger.get_lines_for_level('warning'), [
+            'Option auto_create_account_prefix is deprecated. '
+            'Configure auto_create_account_prefix under the '
+            'swift-constraints section of swift.conf. This option '
+            'will be ignored in a future release.'
+        ])
 
     def test_node_timeout(self):
         # later config should be extended to assert more config options
@@ -1092,9 +1112,11 @@ class TestProxyServer(unittest.TestCase):
 
         self.assertTrue(app.expose_info)
         self.assertIsInstance(app.disallowed_sections, list)
-        self.assertEqual(1, len(app.disallowed_sections))
+        self.assertEqual(2, len(app.disallowed_sections))
-        self.assertEqual(['swift.valid_api_versions'],
+        self.assertEqual([
-                         app.disallowed_sections)
+            'swift.auto_create_account_prefix',
+            'swift.valid_api_versions',
+        ], sorted(app.disallowed_sections))
         self.assertIsNone(app.admin_key)
 
     def test_get_info_controller(self):
@@ -10780,11 +10802,13 @@ class TestSwiftInfo(unittest.TestCase):
         utils._swift_admin_info = {}
 
     def test_registered_defaults(self):
-        proxy_server.Application({}, FakeMemcache(),
+        app = proxy_server.Application({}, FakeMemcache(),
                                        account_ring=FakeRing(),
                                        container_ring=FakeRing())
+        req = Request.blank('/info')
+        resp = req.get_response(app)
+        si = json.loads(resp.body)['swift']
 
-        si = utils.get_swift_info()['swift']
         self.assertIn('version', si)
         self.assertEqual(si['max_file_size'], constraints.MAX_FILE_SIZE)
         self.assertEqual(si['max_meta_name_length'],
@@ -10808,12 +10832,16 @@ class TestSwiftInfo(unittest.TestCase):
         self.assertIn('strict_cors_mode', si)
         self.assertFalse(si['allow_account_management'])
         self.assertFalse(si['account_autocreate'])
-        # This setting is by default excluded by disallowed_sections
-        self.assertEqual(si['valid_api_versions'],
-                         constraints.VALID_API_VERSIONS)
         # this next test is deliberately brittle in order to alert if
         # other items are added to swift info
-        self.assertEqual(len(si), 18)
+        self.assertEqual(len(si), 17)
+
+        si = utils.get_swift_info()['swift']
+        # Tehse settings is by default excluded by disallowed_sections
+        self.assertEqual(si['valid_api_versions'],
+                         constraints.VALID_API_VERSIONS)
+        self.assertEqual(si['auto_create_account_prefix'],
+                         constraints.AUTO_CREATE_ACCOUNT_PREFIX)
 
         self.assertIn('policies', si)
         sorted_pols = sorted(si['policies'], key=operator.itemgetter('name'))