sysnchronize oslo-incubator modules

Update to get jsonutils compatible with Python 3.

The babel Python module is now required by gettextutils, and gettextutils is
imported by excutils.

Change-Id: Ifa6d6b29e2af58dfcfda1ab7efdb2f32cf9de655

oslo/messaging/openstack/common/excutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # Copyright 2012, Red Hat, Inc.
 #
@@ -24,6 +22,8 @@ import sys
 import time
 import traceback
 
+import six
+
 from oslo.messaging.openstack.common.gettextutils import _  # noqa
 
 
@@ -42,13 +42,13 @@ class save_and_reraise_exception(object):
 
     In some cases the caller may not want to re-raise the exception, and
     for those circumstances this context provides a reraise flag that
-    can be used to suppress the exception.  For example:
+    can be used to suppress the exception.  For example::
 
-    except Exception:
+      except Exception:
-        with save_and_reraise_exception() as ctxt:
+          with save_and_reraise_exception() as ctxt:
-            decide_if_need_reraise()
+              decide_if_need_reraise()
-            if not should_be_reraised:
+              if not should_be_reraised:
-                ctxt.reraise = False
+                  ctxt.reraise = False
     """
     def __init__(self):
         self.reraise = True
@@ -65,7 +65,7 @@ class save_and_reraise_exception(object):
                                                      self.tb))
             return False
         if self.reraise:
-            raise self.type_, self.value, self.tb
+            six.reraise(self.type_, self.value, self.tb)
 
 
 def forever_retry_uncaught_exceptions(infunc):
@@ -77,7 +77,8 @@ def forever_retry_uncaught_exceptions(infunc):
             try:
                 return infunc(*args, **kwargs)
             except Exception as exc:
-                if exc.message == last_exc_message:
+                this_exc_message = six.u(str(exc))
+                if this_exc_message == last_exc_message:
                     exc_count += 1
                 else:
                     exc_count = 1
@@ -85,12 +86,12 @@ def forever_retry_uncaught_exceptions(infunc):
                 # the exception message changes
                 cur_time = int(time.time())
                 if (cur_time - last_log_time > 60 or
-                        exc.message != last_exc_message):
+                        this_exc_message != last_exc_message):
                     logging.exception(
                         _('Unexpected exception occurred %d time(s)... '
                           'retrying.') % exc_count)
                     last_log_time = cur_time
-                    last_exc_message = exc.message
+                    last_exc_message = this_exc_message
                     exc_count = 0
                 # This should be a very rare event. In case it isn't, do
                 # a sleep.

oslo/messaging/openstack/common/fixture/moxstubout.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # Copyright 2013 Hewlett-Packard Development Company, L.P.
@@ -19,7 +17,6 @@
 
 import fixtures
 import mox
-import stubout
 
 
 class MoxStubout(fixtures.Fixture):
@@ -30,8 +27,6 @@ class MoxStubout(fixtures.Fixture):
         # emulate some of the mox stuff, we can't use the metaclass
         # because it screws with our generators
         self.mox = mox.Mox()
-        self.stubs = stubout.StubOutForTesting()
+        self.stubs = self.mox.stubs
         self.addCleanup(self.mox.UnsetStubs)
-        self.addCleanup(self.stubs.UnsetAll)
-        self.addCleanup(self.stubs.SmartUnsetAll)
         self.addCleanup(self.mox.VerifyAll)

oslo/messaging/openstack/common/gettextutils.py

@@ -1,8 +1,6 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 Red Hat, Inc.
-# All Rights Reserved.
 # Copyright 2013 IBM Corp.
+# All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -21,27 +19,51 @@ gettext for openstack-common modules.
 
 Usual usage in an openstack.common module:
 
-    from oslo.messaging.openstack.common.gettextutils import _
+    from oslo.messaging.openstack.common.gettextutils import _  # noqa
 """
 
 import copy
 import gettext
-import logging.handlers
+import logging
 import os
 import re
-import UserString
+try:
+    import UserString as _userString
+except ImportError:
+    import collections as _userString
 
+from babel import localedata
 import six
 
 _localedir = os.environ.get('oslo.messaging'.upper() + '_LOCALEDIR')
 _t = gettext.translation('oslo.messaging', localedir=_localedir, fallback=True)
 
+_AVAILABLE_LANGUAGES = {}
+USE_LAZY = False
+
+
+def enable_lazy():
+    """Convenience function for configuring _() to use lazy gettext
+
+    Call this at the start of execution to enable the gettextutils._
+    function to use lazy gettext functionality. This is useful if
+    your project is importing _ directly instead of using the
+    gettextutils.install() way of importing the _ function.
+    """
+    global USE_LAZY
+    USE_LAZY = True
+
 
 def _(msg):
-    return _t.ugettext(msg)
+    if USE_LAZY:
+        return Message(msg, 'oslo.messaging')
+    else:
+        if six.PY3:
+            return _t.gettext(msg)
+        return _t.ugettext(msg)
 
 
-def install(domain):
+def install(domain, lazy=False):
     """Install a _() function using the given translation domain.
 
     Given a translation domain, install a _() function using gettext's
@@ -51,52 +73,60 @@ def install(domain):
     overriding the default localedir (e.g. /usr/share/locale) using
     a translation-domain-specific environment variable (e.g.
     NOVA_LOCALEDIR).
+
+    :param domain: the translation domain
+    :param lazy: indicates whether or not to install the lazy _() function.
+                 The lazy _() introduces a way to do deferred translation
+                 of messages by installing a _ that builds Message objects,
+                 instead of strings, which can then be lazily translated into
+                 any available locale.
     """
-    gettext.install(domain,
+    if lazy:
-                    localedir=os.environ.get(domain.upper() + '_LOCALEDIR'),
+        # NOTE(mrodden): Lazy gettext functionality.
-                    unicode=True)
+        #
+        # The following introduces a deferred way to do translations on
+        # messages in OpenStack. We override the standard _() function
+        # and % (format string) operation to build Message objects that can
+        # later be translated when we have more information.
+        #
+        # Also included below is an example LocaleHandler that translates
+        # Messages to an associated locale, effectively allowing many logs,
+        # each with their own locale.
+
+        def _lazy_gettext(msg):
+            """Create and return a Message object.
+
+            Lazy gettext function for a given domain, it is a factory method
+            for a project/module to get a lazy gettext function for its own
+            translation domain (i.e. nova, glance, cinder, etc.)
+
+            Message encapsulates a string so that we can translate
+            it later when needed.
+            """
+            return Message(msg, domain)
+
+        from six import moves
+        moves.builtins.__dict__['_'] = _lazy_gettext
+    else:
+        localedir = '%s_LOCALEDIR' % domain.upper()
+        if six.PY3:
+            gettext.install(domain,
+                            localedir=os.environ.get(localedir))
+        else:
+            gettext.install(domain,
+                            localedir=os.environ.get(localedir),
+                            unicode=True)
 
 
-"""
+class Message(_userString.UserString, object):
-Lazy gettext functionality.
-
-The following is an attempt to introduce a deferred way
-to do translations on messages in OpenStack. We attempt to
-override the standard _() function and % (format string) operation
-to build Message objects that can later be translated when we have
-more information. Also included is an example LogHandler that
-translates Messages to an associated locale, effectively allowing
-many logs, each with their own locale.
-"""
-
-
-def get_lazy_gettext(domain):
-    """Assemble and return a lazy gettext function for a given domain.
-
-    Factory method for a project/module to get a lazy gettext function
-    for its own translation domain (i.e. nova, glance, cinder, etc.)
-    """
-
-    def _lazy_gettext(msg):
-        """Create and return a Message object.
-
-        Message encapsulates a string so that we can translate it later when
-        needed.
-        """
-        return Message(msg, domain)
-
-    return _lazy_gettext
-
-
-class Message(UserString.UserString, object):
     """Class used to encapsulate translatable messages."""
     def __init__(self, msg, domain):
         # _msg is the gettext msgid and should never change
         self._msg = msg
         self._left_extra_msg = ''
         self._right_extra_msg = ''
+        self._locale = None
         self.params = None
-        self.locale = None
         self.domain = domain
 
     @property
@@ -116,8 +146,13 @@ class Message(UserString.UserString, object):
                                        localedir=localedir,
                                        fallback=True)
 
+        if six.PY3:
+            ugettext = lang.gettext
+        else:
+            ugettext = lang.ugettext
+
         full_msg = (self._left_extra_msg +
-                    lang.ugettext(self._msg) +
+                    ugettext(self._msg) +
                     self._right_extra_msg)
 
         if self.params is not None:
@@ -125,12 +160,39 @@ class Message(UserString.UserString, object):
 
         return six.text_type(full_msg)
 
+    @property
+    def locale(self):
+        return self._locale
+
+    @locale.setter
+    def locale(self, value):
+        self._locale = value
+        if not self.params:
+            return
+
+        # This Message object may have been constructed with one or more
+        # Message objects as substitution parameters, given as a single
+        # Message, or a tuple or Map containing some, so when setting the
+        # locale for this Message we need to set it for those Messages too.
+        if isinstance(self.params, Message):
+            self.params.locale = value
+            return
+        if isinstance(self.params, tuple):
+            for param in self.params:
+                if isinstance(param, Message):
+                    param.locale = value
+            return
+        if isinstance(self.params, dict):
+            for param in self.params.values():
+                if isinstance(param, Message):
+                    param.locale = value
+
     def _save_dictionary_parameter(self, dict_param):
         full_msg = self.data
         # look for %(blah) fields in string;
         # ignore %% and deal with the
         # case where % is first character on the line
-        keys = re.findall('(?:[^%]|^)%\((\w*)\)[a-z]', full_msg)
+        keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', full_msg)
 
         # if we don't find any %(blah) blocks but have a %s
         if not keys and re.findall('(?:[^%]|^)%[a-z]', full_msg):
@@ -143,7 +205,7 @@ class Message(UserString.UserString, object):
                     params[key] = copy.deepcopy(dict_param[key])
                 except TypeError:
                     # cast uncopyable thing to unicode string
-                    params[key] = unicode(dict_param[key])
+                    params[key] = six.text_type(dict_param[key])
 
         return params
 
@@ -162,7 +224,7 @@ class Message(UserString.UserString, object):
             try:
                 self.params = copy.deepcopy(other)
             except TypeError:
-                self.params = unicode(other)
+                self.params = six.text_type(other)
 
         return self
 
@@ -171,11 +233,13 @@ class Message(UserString.UserString, object):
         return self.data
 
     def __str__(self):
+        if six.PY3:
+            return self.__unicode__()
         return self.data.encode('utf-8')
 
     def __getstate__(self):
         to_copy = ['_msg', '_right_extra_msg', '_left_extra_msg',
-                   'domain', 'params', 'locale']
+                   'domain', 'params', '_locale']
         new_dict = self.__dict__.fromkeys(to_copy)
         for attr in to_copy:
             new_dict[attr] = copy.deepcopy(self.__dict__[attr])
@@ -229,7 +293,55 @@ class Message(UserString.UserString, object):
         if name in ops:
             return getattr(self.data, name)
         else:
-            return UserString.UserString.__getattribute__(self, name)
+            return _userString.UserString.__getattribute__(self, name)
+
+
+def get_available_languages(domain):
+    """Lists the available languages for the given translation domain.
+
+    :param domain: the domain to get languages for
+    """
+    if domain in _AVAILABLE_LANGUAGES:
+        return copy.copy(_AVAILABLE_LANGUAGES[domain])
+
+    localedir = '%s_LOCALEDIR' % domain.upper()
+    find = lambda x: gettext.find(domain,
+                                  localedir=os.environ.get(localedir),
+                                  languages=[x])
+
+    # NOTE(mrodden): en_US should always be available (and first in case
+    # order matters) since our in-line message strings are en_US
+    language_list = ['en_US']
+    # NOTE(luisg): Babel <1.0 used a function called list(), which was
+    # renamed to locale_identifiers() in >=1.0, the requirements master list
+    # requires >=0.9.6, uncapped, so defensively work with both. We can remove
+    # this check when the master list updates to >=1.0, and update all projects
+    list_identifiers = (getattr(localedata, 'list', None) or
+                        getattr(localedata, 'locale_identifiers'))
+    locale_identifiers = list_identifiers()
+    for i in locale_identifiers:
+        if find(i) is not None:
+            language_list.append(i)
+    _AVAILABLE_LANGUAGES[domain] = language_list
+    return copy.copy(language_list)
+
+
+def get_localized_message(message, user_locale):
+    """Gets a localized version of the given message in the given locale.
+
+    If the message is not a Message object the message is returned as-is.
+    If the locale is None the message is translated to the default locale.
+
+    :returns: the translated message in unicode, or the original message if
+              it could not be translated
+    """
+    translated = message
+    if isinstance(message, Message):
+        original_locale = message.locale
+        message.locale = user_locale
+        translated = six.text_type(message)
+        message.locale = original_locale
+    return translated
 
 
 class LocaleHandler(logging.Handler):

oslo/messaging/openstack/common/importutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #

oslo/messaging/openstack/common/jsonutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # Copyright 2011 Justin Santa Barbara
@@ -38,13 +36,23 @@ import functools
 import inspect
 import itertools
 import json
-import types
+try:
-import xmlrpclib
+    import xmlrpclib
+except ImportError:
+    # NOTE(jaypipes): xmlrpclib was renamed to xmlrpc.client in Python3
+    #                 however the function and object call signatures
+    #                 remained the same. This whole try/except block should
+    #                 be removed and replaced with a call to six.moves once
+    #                 six 1.4.2 is released. See http://bit.ly/1bqrVzu
+    import xmlrpc.client as xmlrpclib
 
 import six
 
+from oslo.messaging.openstack.common import gettextutils
+from oslo.messaging.openstack.common import importutils
 from oslo.messaging.openstack.common import timeutils
 
+netaddr = importutils.try_import("netaddr")
 
 _nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
                      inspect.isfunction, inspect.isgeneratorfunction,
@@ -52,7 +60,8 @@ _nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
                      inspect.iscode, inspect.isbuiltin, inspect.isroutine,
                      inspect.isabstract]
 
-_simple_types = (types.NoneType, int, basestring, bool, float, long)
+_simple_types = (six.string_types + six.integer_types
+                 + (type(None), bool, float))
 
 
 def to_primitive(value, convert_instances=False, convert_datetime=True,
@@ -117,7 +126,7 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
                                       level=level,
                                       max_depth=max_depth)
         if isinstance(value, dict):
-            return dict((k, recursive(v)) for k, v in value.iteritems())
+            return dict((k, recursive(v)) for k, v in six.iteritems(value))
         elif isinstance(value, (list, tuple)):
             return [recursive(lv) for lv in value]
 
@@ -129,6 +138,8 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
 
         if convert_datetime and isinstance(value, datetime.datetime):
             return timeutils.strtime(value)
+        elif isinstance(value, gettextutils.Message):
+            return value.data
         elif hasattr(value, 'iteritems'):
             return recursive(dict(value.iteritems()), level=level + 1)
         elif hasattr(value, '__iter__'):
@@ -137,6 +148,8 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
             # Likely an instance of something. Watch for cycles.
             # Ignore class member vars.
             return recursive(value.__dict__, level=level + 1)
+        elif netaddr and isinstance(value, netaddr.IPAddress):
+            return six.text_type(value)
         else:
             if any(test(value) for test in _nasty_type_tests):
                 return six.text_type(value)

oslo/messaging/openstack/common/network_utils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -19,7 +17,7 @@
 Network-related utilities and helper functions.
 """
 
-import urlparse
+from oslo.messaging.openstack.common.py3kcompat import urlutils
 
 
 def parse_host_port(address, default_port=None):
@@ -72,10 +70,10 @@ def urlsplit(url, scheme='', allow_fragments=True):
 
     The parameters are the same as urlparse.urlsplit.
     """
-    scheme, netloc, path, query, fragment = urlparse.urlsplit(
+    scheme, netloc, path, query, fragment = urlutils.urlsplit(
         url, scheme, allow_fragments)
     if allow_fragments and '#' in path:
         path, fragment = path.split('#', 1)
     if '?' in path:
         path, query = path.split('?', 1)
-    return urlparse.SplitResult(scheme, netloc, path, query, fragment)
+    return urlutils.SplitResult(scheme, netloc, path, query, fragment)

oslo/messaging/openstack/common/sslutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2013 IBM Corp.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

oslo/messaging/openstack/common/timeutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -21,8 +19,10 @@ Time related utilities and helper functions.
 
 import calendar
 import datetime
+import time
 
 import iso8601
+import six
 
 
 # ISO 8601 extended time format with microseconds
@@ -48,9 +48,9 @@ def parse_isotime(timestr):
     try:
         return iso8601.parse_date(timestr)
     except iso8601.ParseError as e:
-        raise ValueError(e.message)
+        raise ValueError(six.text_type(e))
     except TypeError as e:
-        raise ValueError(e.message)
+        raise ValueError(six.text_type(e))
 
 
 def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
@@ -75,20 +75,31 @@ def normalize_time(timestamp):
 
 def is_older_than(before, seconds):
     """Return True if before is older than seconds."""
-    if isinstance(before, basestring):
+    if isinstance(before, six.string_types):
         before = parse_strtime(before).replace(tzinfo=None)
+    else:
+        before = before.replace(tzinfo=None)
+
     return utcnow() - before > datetime.timedelta(seconds=seconds)
 
 
 def is_newer_than(after, seconds):
     """Return True if after is newer than seconds."""
-    if isinstance(after, basestring):
+    if isinstance(after, six.string_types):
         after = parse_strtime(after).replace(tzinfo=None)
+    else:
+        after = after.replace(tzinfo=None)
+
     return after - utcnow() > datetime.timedelta(seconds=seconds)
 
 
 def utcnow_ts():
     """Timestamp version of our utcnow function."""
+    if utcnow.override_time is None:
+        # NOTE(kgriffs): This is several times faster
+        # than going through calendar.timegm(...)
+        return int(time.time())
+
     return calendar.timegm(utcnow().timetuple())
 
 
@@ -110,12 +121,15 @@ def iso8601_from_timestamp(timestamp):
 utcnow.override_time = None
 
 
-def set_time_override(override_time=datetime.datetime.utcnow()):
+def set_time_override(override_time=None):
     """Overrides utils.utcnow.
 
     Make it return a constant time or a list thereof, one at a time.
+
+    :param override_time: datetime instance or list thereof. If not
+                          given, defaults to the current UTC time.
     """
-    utcnow.override_time = override_time
+    utcnow.override_time = override_time or datetime.datetime.utcnow()
 
 
 def advance_time_delta(timedelta):
@@ -168,6 +182,15 @@ def delta_seconds(before, after):
     datetime objects (as a float, to microsecond resolution).
     """
     delta = after - before
+    return total_seconds(delta)
+
+
+def total_seconds(delta):
+    """Return the total seconds of datetime.timedelta object.
+
+    Compute total seconds of datetime.timedelta, datetime.timedelta
+    doesn't have method total_seconds in Python2.6, calculate it manually.
+    """
     try:
         return delta.total_seconds()
     except AttributeError:

requirements.txt

@@ -10,3 +10,6 @@ six>=1.4.1
 # FIXME(markmc): remove this when the drivers no longer
 # import eventlet
 eventlet>=0.13.0
+
+# used by openstack/common/gettextutils.py
+Babel>=1.3