Update oslo

Update gettext, striutils, timeutils and install_venv_common
from oslo.

Change-Id: Ibd9067e3e2be335ef75f0e4a5e4000d143030ab7
Signed-off-by: Chuck Short <chuck.short@canonical.com>

novaclient/openstack/common/gettextutils.py

@@ -1,6 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2012 Red Hat, Inc.
+# Copyright 2013 IBM Corp.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -23,18 +24,27 @@ Usual usage in an openstack.common module:
     from novaclient.openstack.common.gettextutils import _
 """
 
+import copy
 import gettext
+import logging.handlers
 import os
+import re
+import UserString
+
+from babel import localedata
+import six
 
 _localedir = os.environ.get('novaclient'.upper() + '_LOCALEDIR')
 _t = gettext.translation('novaclient', localedir=_localedir, fallback=True)
 
+_AVAILABLE_LANGUAGES = []
+
 
 def _(msg):
     return _t.ugettext(msg)
 
 
-def install(domain):
+def install(domain, lazy=False):
     """Install a _() function using the given translation domain.
 
     Given a translation domain, install a _() function using gettext's
@@ -44,7 +54,252 @@ def install(domain):
     overriding the default localedir (e.g. /usr/share/locale) using
     a translation-domain-specific environment variable (e.g.
     NOVA_LOCALEDIR).
+
+    :param domain: the translation domain
+    :param lazy: indicates whether or not to install the lazy _() function.
+                 The lazy _() introduces a way to do deferred translation
+                 of messages by installing a _ that builds Message objects,
+                 instead of strings, which can then be lazily translated into
+                 any available locale.
     """
-    gettext.install(domain,
+    if lazy:
-                    localedir=os.environ.get(domain.upper() + '_LOCALEDIR'),
+        # NOTE(mrodden): Lazy gettext functionality.
-                    unicode=True)
+        #
+        # The following introduces a deferred way to do translations on
+        # messages in OpenStack. We override the standard _() function
+        # and % (format string) operation to build Message objects that can
+        # later be translated when we have more information.
+        #
+        # Also included below is an example LocaleHandler that translates
+        # Messages to an associated locale, effectively allowing many logs,
+        # each with their own locale.
+
+        def _lazy_gettext(msg):
+            """Create and return a Message object.
+
+            Lazy gettext function for a given domain, it is a factory method
+            for a project/module to get a lazy gettext function for its own
+            translation domain (i.e. nova, glance, cinder, etc.)
+
+            Message encapsulates a string so that we can translate
+            it later when needed.
+            """
+            return Message(msg, domain)
+
+        import __builtin__
+        __builtin__.__dict__['_'] = _lazy_gettext
+    else:
+        localedir = '%s_LOCALEDIR' % domain.upper()
+        gettext.install(domain,
+                        localedir=os.environ.get(localedir),
+                        unicode=True)
+
+
+class Message(UserString.UserString, object):
+    """Class used to encapsulate translatable messages."""
+    def __init__(self, msg, domain):
+        # _msg is the gettext msgid and should never change
+        self._msg = msg
+        self._left_extra_msg = ''
+        self._right_extra_msg = ''
+        self.params = None
+        self.locale = None
+        self.domain = domain
+
+    @property
+    def data(self):
+        # NOTE(mrodden): this should always resolve to a unicode string
+        # that best represents the state of the message currently
+
+        localedir = os.environ.get(self.domain.upper() + '_LOCALEDIR')
+        if self.locale:
+            lang = gettext.translation(self.domain,
+                                       localedir=localedir,
+                                       languages=[self.locale],
+                                       fallback=True)
+        else:
+            # use system locale for translations
+            lang = gettext.translation(self.domain,
+                                       localedir=localedir,
+                                       fallback=True)
+
+        full_msg = (self._left_extra_msg +
+                    lang.ugettext(self._msg) +
+                    self._right_extra_msg)
+
+        if self.params is not None:
+            full_msg = full_msg % self.params
+
+        return six.text_type(full_msg)
+
+    def _save_dictionary_parameter(self, dict_param):
+        full_msg = self.data
+        # look for %(blah) fields in string;
+        # ignore %% and deal with the
+        # case where % is first character on the line
+        keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', full_msg)
+
+        # if we don't find any %(blah) blocks but have a %s
+        if not keys and re.findall('(?:[^%]|^)%[a-z]', full_msg):
+            # apparently the full dictionary is the parameter
+            params = copy.deepcopy(dict_param)
+        else:
+            params = {}
+            for key in keys:
+                try:
+                    params[key] = copy.deepcopy(dict_param[key])
+                except TypeError:
+                    # cast uncopyable thing to unicode string
+                    params[key] = unicode(dict_param[key])
+
+        return params
+
+    def _save_parameters(self, other):
+        # we check for None later to see if
+        # we actually have parameters to inject,
+        # so encapsulate if our parameter is actually None
+        if other is None:
+            self.params = (other, )
+        elif isinstance(other, dict):
+            self.params = self._save_dictionary_parameter(other)
+        else:
+            # fallback to casting to unicode,
+            # this will handle the problematic python code-like
+            # objects that cannot be deep-copied
+            try:
+                self.params = copy.deepcopy(other)
+            except TypeError:
+                self.params = unicode(other)
+
+        return self
+
+    # overrides to be more string-like
+    def __unicode__(self):
+        return self.data
+
+    def __str__(self):
+        return self.data.encode('utf-8')
+
+    def __getstate__(self):
+        to_copy = ['_msg', '_right_extra_msg', '_left_extra_msg',
+                   'domain', 'params', 'locale']
+        new_dict = self.__dict__.fromkeys(to_copy)
+        for attr in to_copy:
+            new_dict[attr] = copy.deepcopy(self.__dict__[attr])
+
+        return new_dict
+
+    def __setstate__(self, state):
+        for (k, v) in state.items():
+            setattr(self, k, v)
+
+    # operator overloads
+    def __add__(self, other):
+        copied = copy.deepcopy(self)
+        copied._right_extra_msg += other.__str__()
+        return copied
+
+    def __radd__(self, other):
+        copied = copy.deepcopy(self)
+        copied._left_extra_msg += other.__str__()
+        return copied
+
+    def __mod__(self, other):
+        # do a format string to catch and raise
+        # any possible KeyErrors from missing parameters
+        self.data % other
+        copied = copy.deepcopy(self)
+        return copied._save_parameters(other)
+
+    def __mul__(self, other):
+        return self.data * other
+
+    def __rmul__(self, other):
+        return other * self.data
+
+    def __getitem__(self, key):
+        return self.data[key]
+
+    def __getslice__(self, start, end):
+        return self.data.__getslice__(start, end)
+
+    def __getattribute__(self, name):
+        # NOTE(mrodden): handle lossy operations that we can't deal with yet
+        # These override the UserString implementation, since UserString
+        # uses our __class__ attribute to try and build a new message
+        # after running the inner data string through the operation.
+        # At that point, we have lost the gettext message id and can just
+        # safely resolve to a string instead.
+        ops = ['capitalize', 'center', 'decode', 'encode',
+               'expandtabs', 'ljust', 'lstrip', 'replace', 'rjust', 'rstrip',
+               'strip', 'swapcase', 'title', 'translate', 'upper', 'zfill']
+        if name in ops:
+            return getattr(self.data, name)
+        else:
+            return UserString.UserString.__getattribute__(self, name)
+
+
+def get_available_languages(domain):
+    """Lists the available languages for the given translation domain.
+
+    :param domain: the domain to get languages for
+    """
+    if _AVAILABLE_LANGUAGES:
+        return _AVAILABLE_LANGUAGES
+
+    localedir = '%s_LOCALEDIR' % domain.upper()
+    find = lambda x: gettext.find(domain,
+                                  localedir=os.environ.get(localedir),
+                                  languages=[x])
+
+    # NOTE(mrodden): en_US should always be available (and first in case
+    # order matters) since our in-line message strings are en_US
+    _AVAILABLE_LANGUAGES.append('en_US')
+    # NOTE(luisg): Babel <1.0 used a function called list(), which was
+    # renamed to locale_identifiers() in >=1.0, the requirements master list
+    # requires >=0.9.6, uncapped, so defensively work with both. We can remove
+    # this check when the master list updates to >=1.0, and all projects udpate
+    list_identifiers = (getattr(localedata, 'list', None) or
+                        getattr(localedata, 'locale_identifiers'))
+    locale_identifiers = list_identifiers()
+    for i in locale_identifiers:
+        if find(i) is not None:
+            _AVAILABLE_LANGUAGES.append(i)
+    return _AVAILABLE_LANGUAGES
+
+
+def get_localized_message(message, user_locale):
+    """Gets a localized version of the given message in the given locale."""
+    if (isinstance(message, Message)):
+        if user_locale:
+            message.locale = user_locale
+        return unicode(message)
+    else:
+        return message
+
+
+class LocaleHandler(logging.Handler):
+    """Handler that can have a locale associated to translate Messages.
+
+    A quick example of how to utilize the Message class above.
+    LocaleHandler takes a locale and a target logging.Handler object
+    to forward LogRecord objects to after translating the internal Message.
+    """
+
+    def __init__(self, locale, target):
+        """Initialize a LocaleHandler
+
+        :param locale: locale to use for translating messages
+        :param target: logging.Handler object to forward
+                       LogRecord objects to after translation
+        """
+        logging.Handler.__init__(self)
+        self.locale = locale
+        self.target = target
+
+    def emit(self, record):
+        if isinstance(record.msg, Message):
+            # set the locale and resolve to a string
+            record.msg.locale = self.locale
+
+        self.target.emit(record)

novaclient/openstack/common/strutils.py

@@ -19,18 +19,35 @@
 System-level utilities and helper functions.
 """
 
+import re
 import sys
+import unicodedata
 
-from novaclient.openstack.common.gettextutils import _
+import six
 
+from novaclient.openstack.common.gettextutils import _  # noqa
+
+
+# Used for looking up extensions of text
+# to their 'multiplied' byte amount
+BYTE_MULTIPLIERS = {
+    '': 1,
+    't': 1024 ** 4,
+    'g': 1024 ** 3,
+    'm': 1024 ** 2,
+    'k': 1024,
+}
+BYTE_REGEX = re.compile(r'(^-?\d+)(\D*)')
 
 TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
 FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')
 
+SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
+SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")
+
 
 def int_from_bool_as_string(subject):
-    """
+    """Interpret a string as a boolean and return either 1 or 0.
-    Interpret a string as a boolean and return either 1 or 0.
 
     Any string value in:
 
@@ -44,8 +61,7 @@ def int_from_bool_as_string(subject):
 
 
 def bool_from_string(subject, strict=False):
-    """
+    """Interpret a string as a boolean.
-    Interpret a string as a boolean.
 
     A case-insensitive match is performed such that strings matching 't',
     'true', 'on', 'y', 'yes', or '1' are considered True and, when
@@ -57,7 +73,7 @@ def bool_from_string(subject, strict=False):
     ValueError which is useful when parsing values passed in from an API call.
     Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
     """
-    if not isinstance(subject, basestring):
+    if not isinstance(subject, six.string_types):
         subject = str(subject)
 
     lowered = subject.strip().lower()
@@ -78,21 +94,19 @@ def bool_from_string(subject, strict=False):
 
 
 def safe_decode(text, incoming=None, errors='strict'):
-    """
+    """Decodes incoming str using `incoming` if they're not already unicode.
-    Decodes incoming str using `incoming` if they're
-    not already unicode.
 
     :param incoming: Text's current encoding
     :param errors: Errors handling policy. See here for valid
         values http://docs.python.org/2/library/codecs.html
     :returns: text or a unicode `incoming` encoded
                 representation of it.
-    :raises TypeError: If text is not an isntance of basestring
+    :raises TypeError: If text is not an isntance of str
     """
-    if not isinstance(text, basestring):
+    if not isinstance(text, six.string_types):
         raise TypeError("%s can't be decoded" % type(text))
 
-    if isinstance(text, unicode):
+    if isinstance(text, six.text_type):
         return text
 
     if not incoming:
@@ -119,11 +133,10 @@ def safe_decode(text, incoming=None, errors='strict'):
 
 def safe_encode(text, incoming=None,
                 encoding='utf-8', errors='strict'):
-    """
+    """Encodes incoming str/unicode using `encoding`.
-    Encodes incoming str/unicode using `encoding`. If
+
-    incoming is not specified, text is expected to
+    If incoming is not specified, text is expected to be encoded with
-    be encoded with current python's default encoding.
+    current python's default encoding. (`sys.getdefaultencoding`)
-    (`sys.getdefaultencoding`)
 
     :param incoming: Text's current encoding
     :param encoding: Expected encoding for text (Default UTF-8)
@@ -131,16 +144,16 @@ def safe_encode(text, incoming=None,
         values http://docs.python.org/2/library/codecs.html
     :returns: text or a bytestring `encoding` encoded
                 representation of it.
-    :raises TypeError: If text is not an isntance of basestring
+    :raises TypeError: If text is not an isntance of str
     """
-    if not isinstance(text, basestring):
+    if not isinstance(text, six.string_types):
         raise TypeError("%s can't be encoded" % type(text))
 
     if not incoming:
         incoming = (sys.stdin.encoding or
                     sys.getdefaultencoding())
 
-    if isinstance(text, unicode):
+    if isinstance(text, six.text_type):
         return text.encode(encoding, errors)
     elif text and encoding != incoming:
         # Decode text before encoding it with `encoding`
@@ -148,3 +161,58 @@ def safe_encode(text, incoming=None,
         return text.encode(encoding, errors)
 
     return text
+
+
+def to_bytes(text, default=0):
+    """Converts a string into an integer of bytes.
+
+    Looks at the last characters of the text to determine
+    what conversion is needed to turn the input text into a byte number.
+    Supports "B, K(B), M(B), G(B), and T(B)". (case insensitive)
+
+    :param text: String input for bytes size conversion.
+    :param default: Default return value when text is blank.
+
+    """
+    match = BYTE_REGEX.search(text)
+    if match:
+        magnitude = int(match.group(1))
+        mult_key_org = match.group(2)
+        if not mult_key_org:
+            return magnitude
+    elif text:
+        msg = _('Invalid string format: %s') % text
+        raise TypeError(msg)
+    else:
+        return default
+    mult_key = mult_key_org.lower().replace('b', '', 1)
+    multiplier = BYTE_MULTIPLIERS.get(mult_key)
+    if multiplier is None:
+        msg = _('Unknown byte multiplier: %s') % mult_key_org
+        raise TypeError(msg)
+    return magnitude * multiplier
+
+
+def to_slug(value, incoming=None, errors="strict"):
+    """Normalize string.
+
+    Convert to lowercase, remove non-word characters, and convert spaces
+    to hyphens.
+
+    Inspired by Django's `slugify` filter.
+
+    :param value: Text to slugify
+    :param incoming: Text's current encoding
+    :param errors: Errors handling policy. See here for valid
+        values http://docs.python.org/2/library/codecs.html
+    :returns: slugified unicode representation of `value`
+    :raises TypeError: If text is not an instance of str
+    """
+    value = safe_decode(value, incoming, errors)
+    # NOTE(aababilov): no need to use safe_(encode|decode) here:
+    # encodings are always "ascii", error handling is always "ignore"
+    # and types are always known (first: unicode; second: str)
+    value = unicodedata.normalize("NFKD", value).encode(
+        "ascii", "ignore").decode("ascii")
+    value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
+    return SLUGIFY_HYPHENATE_RE.sub("-", value)

novaclient/openstack/common/timeutils.py

@@ -23,6 +23,7 @@ import calendar
 import datetime
 
 import iso8601
+import six
 
 
 # ISO 8601 extended time format with microseconds
@@ -32,7 +33,7 @@ PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND
 
 
 def isotime(at=None, subsecond=False):
-    """Stringify time in ISO 8601 format"""
+    """Stringify time in ISO 8601 format."""
     if not at:
         at = utcnow()
     st = at.strftime(_ISO8601_TIME_FORMAT
@@ -44,13 +45,13 @@ def isotime(at=None, subsecond=False):
 
 
 def parse_isotime(timestr):
-    """Parse time from ISO 8601 format"""
+    """Parse time from ISO 8601 format."""
     try:
         return iso8601.parse_date(timestr)
     except iso8601.ParseError as e:
-        raise ValueError(e.message)
+        raise ValueError(unicode(e))
     except TypeError as e:
-        raise ValueError(e.message)
+        raise ValueError(unicode(e))
 
 
 def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
@@ -66,7 +67,7 @@ def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
 
 
 def normalize_time(timestamp):
-    """Normalize time in arbitrary timezone to UTC naive object"""
+    """Normalize time in arbitrary timezone to UTC naive object."""
     offset = timestamp.utcoffset()
     if offset is None:
         return timestamp
@@ -75,14 +76,14 @@ def normalize_time(timestamp):
 
 def is_older_than(before, seconds):
     """Return True if before is older than seconds."""
-    if isinstance(before, basestring):
+    if isinstance(before, six.string_types):
         before = parse_strtime(before).replace(tzinfo=None)
     return utcnow() - before > datetime.timedelta(seconds=seconds)
 
 
 def is_newer_than(after, seconds):
     """Return True if after is newer than seconds."""
-    if isinstance(after, basestring):
+    if isinstance(after, six.string_types):
         after = parse_strtime(after).replace(tzinfo=None)
     return after - utcnow() > datetime.timedelta(seconds=seconds)
 
@@ -103,7 +104,7 @@ def utcnow():
 
 
 def iso8601_from_timestamp(timestamp):
-    """Returns a iso8601 formated date from timestamp"""
+    """Returns a iso8601 formated date from timestamp."""
     return isotime(datetime.datetime.utcfromtimestamp(timestamp))
 
 
@@ -111,9 +112,9 @@ utcnow.override_time = None
 
 
 def set_time_override(override_time=datetime.datetime.utcnow()):
-    """
+    """Overrides utils.utcnow.
-    Override utils.utcnow to return a constant time or a list thereof,
+
-    one at a time.
+    Make it return a constant time or a list thereof, one at a time.
     """
     utcnow.override_time = override_time
 
@@ -141,7 +142,8 @@ def clear_time_override():
 def marshall_now(now=None):
     """Make an rpc-safe datetime with microseconds.
 
-    Note: tzinfo is stripped, but not required for relative times."""
+    Note: tzinfo is stripped, but not required for relative times.
+    """
     if not now:
         now = utcnow()
     return dict(day=now.day, month=now.month, year=now.year, hour=now.hour,
@@ -161,7 +163,8 @@ def unmarshall_time(tyme):
 
 
 def delta_seconds(before, after):
-    """
+    """Return the difference between two timing objects.
+
     Compute the difference in seconds between two date, time, or
     datetime objects (as a float, to microsecond resolution).
     """
@@ -174,8 +177,7 @@ def delta_seconds(before, after):
 
 
 def is_soon(dt, window):
-    """
+    """Determines if time is going to happen in the next window seconds.
-    Determines if time is going to happen in the next window seconds.
 
     :params dt: the time
     :params window: minimum seconds to remain to consider the time not soon

requirements.txt

@@ -5,3 +5,4 @@ PrettyTable>=0.6,<0.8
 requests>=1.1
 simplejson>=2.0.9
 six
+Babel>=0.9.6

tools/install_venv_common.py

@@ -114,9 +114,10 @@ class InstallVenv(object):
         print('Installing dependencies with pip (this can take a while)...')
 
         # First things first, make sure our venv has the latest pip and
-        # setuptools.
+        # setuptools and pbr
-        self.pip_install('pip>=1.3')
+        self.pip_install('pip>=1.4')
         self.pip_install('setuptools')
+        self.pip_install('pbr')
 
         self.pip_install('-r', self.requirements)
         self.pip_install('-r', self.test_requirements)
@@ -201,12 +202,13 @@ class Fedora(Distro):
         RHEL: https://bugzilla.redhat.com/958868
         """
 
-        # Install "patch" program if it's not there
+        if os.path.exists('contrib/redhat-eventlet.patch'):
-        if not self.check_pkg('patch'):
+            # Install "patch" program if it's not there
-            self.die("Please install 'patch'.")
+            if not self.check_pkg('patch'):
+                self.die("Please install 'patch'.")
 
-        # Apply the eventlet patch
+            # Apply the eventlet patch
-        self.apply_patch(os.path.join(self.venv, 'lib', self.py_version,
+            self.apply_patch(os.path.join(self.venv, 'lib', self.py_version,
-                                      'site-packages',
+                                          'site-packages',
-                                      'eventlet/green/subprocess.py'),
+                                          'eventlet/green/subprocess.py'),
-                         'contrib/redhat-eventlet.patch')
+                             'contrib/redhat-eventlet.patch')