From f881a6242d644e5b25bef94c3badf22faea6e85f Mon Sep 17 00:00:00 2001
From: Davanum Srinivas <dims@linux.vnet.ibm.com>
Date: Tue, 28 Oct 2014 07:53:06 -0400
Subject: [PATCH] Sync latest from oslo-incubator

Changes include:
1523f00 threadgroup: don't log GreenletExit
a99c5f1 Report: Support Sequences and Mappings as Data
4ca5091 Fixes nits in module policy
411ba35 Simplify is_user_context method
5d43644 cliutils restore python3 string decode
a3220c5 add list_opts to all modules with configuration options
df774ff Import PublishErrorsHandler from oslo.messaging
d0a1fde Switch fileutils to use python logging
b2d35ee Use list.pop(0) to keep the code simpler
262fc82 Correct default rule name for policy.Enforcer
046bc2a Mask secrets when output conf

Removed fixtures that are no longer in oslo-incubator as the
previous reviews in this series switched to using the equivalent
fixtures in oslo libraries.

Cleaned up openstack-common.conf to include only the modules
that are still in oslo-incubator.

Change-Id: Ifd2d5bd506aeb2a9400c5075c3ca24b3c02d1cb6
---
 nova/openstack/common/cliutils.py             | 11 ++++++++--
 nova/openstack/common/context.py              |  8 ++-----
 nova/openstack/common/eventlet_backdoor.py    |  7 ++++++
 nova/openstack/common/fileutils.py            |  3 +--
 nova/openstack/common/imageutils.py           |  5 ++---
 nova/openstack/common/log.py                  | 22 ++++++++++++-------
 nova/openstack/common/memorycache.py          |  7 ++++++
 nova/openstack/common/periodic_task.py        |  6 +++++
 nova/openstack/common/policy.py               | 16 +++++++++-----
 .../common/report/generators/threading.py     |  6 ++---
 nova/openstack/common/report/models/base.py   | 22 +++++++++++++++++--
 nova/openstack/common/report/models/conf.py   | 16 ++++++++++----
 nova/openstack/common/sslutils.py             | 10 +++++++--
 nova/openstack/common/threadgroup.py          |  2 ++
 openstack-common.conf                         |  7 ------
 15 files changed, 103 insertions(+), 45 deletions(-)

diff --git a/nova/openstack/common/cliutils.py b/nova/openstack/common/cliutils.py
index aa9354f7c0da..41eab0ecf2f2 100644
--- a/nova/openstack/common/cliutils.py
+++ b/nova/openstack/common/cliutils.py
@@ -180,7 +180,10 @@ def print_list(objs, fields, formatters=None, sortby_index=0,
                 row.append(data)
         pt.add_row(row)
 
-    print(encodeutils.safe_encode(pt.get_string(**kwargs)))
+    if six.PY3:
+        print(encodeutils.safe_encode(pt.get_string(**kwargs)).decode())
+    else:
+        print(encodeutils.safe_encode(pt.get_string(**kwargs)))
 
 
 def print_dict(dct, dict_property="Property", wrap=0):
@@ -208,7 +211,11 @@ def print_dict(dct, dict_property="Property", wrap=0):
                 col1 = ''
         else:
             pt.add_row([k, v])
-    print(encodeutils.safe_encode(pt.get_string()))
+
+    if six.PY3:
+        print(encodeutils.safe_encode(pt.get_string()).decode())
+    else:
+        print(encodeutils.safe_encode(pt.get_string()))
 
 
 def get_password(max_password_prompts=3):
diff --git a/nova/openstack/common/context.py b/nova/openstack/common/context.py
index b612db71407c..168989004c22 100644
--- a/nova/openstack/common/context.py
+++ b/nova/openstack/common/context.py
@@ -117,10 +117,6 @@ def get_context_from_function_and_args(function, args, kwargs):
 
 def is_user_context(context):
     """Indicates if the request context is a normal user."""
-    if not context:
+    if not context or context.is_admin:
         return False
-    if context.is_admin:
-        return False
-    if not context.user_id or not context.project_id:
-        return False
-    return True
+    return context.user_id and context.project_id
diff --git a/nova/openstack/common/eventlet_backdoor.py b/nova/openstack/common/eventlet_backdoor.py
index ec0d6eb04614..ef1f1ee351c4 100644
--- a/nova/openstack/common/eventlet_backdoor.py
+++ b/nova/openstack/common/eventlet_backdoor.py
@@ -16,6 +16,7 @@
 
 from __future__ import print_function
 
+import copy
 import errno
 import gc
 import os
@@ -49,6 +50,12 @@ CONF.register_opts(eventlet_backdoor_opts)
 LOG = logging.getLogger(__name__)
 
 
+def list_opts():
+    """Entry point for oslo.config-generator.
+    """
+    return [(None, copy.deepcopy(eventlet_backdoor_opts))]
+
+
 class EventletBackdoorConfigValueError(Exception):
     def __init__(self, port_range, help_msg, ex):
         msg = ('Invalid backdoor_port configuration %(range)s: %(ex)s. '
diff --git a/nova/openstack/common/fileutils.py b/nova/openstack/common/fileutils.py
index 617a094986f6..ec26eaf9cde4 100644
--- a/nova/openstack/common/fileutils.py
+++ b/nova/openstack/common/fileutils.py
@@ -15,13 +15,12 @@
 
 import contextlib
 import errno
+import logging
 import os
 import tempfile
 
 from oslo.utils import excutils
 
-from nova.openstack.common import log as logging
-
 LOG = logging.getLogger(__name__)
 
 _FILE_CACHE = {}
diff --git a/nova/openstack/common/imageutils.py b/nova/openstack/common/imageutils.py
index d7bd8256f388..1e9eef2b880e 100644
--- a/nova/openstack/common/imageutils.py
+++ b/nova/openstack/common/imageutils.py
@@ -101,10 +101,9 @@ class QemuImgInfo(object):
             real_details = real_details.strip().lower()
         elif root_cmd == 'snapshot_list':
             # Next line should be a header, starting with 'ID'
-            if not lines_after or not lines_after[0].startswith("ID"):
+            if not lines_after or not lines_after.pop(0).startswith("ID"):
                 msg = _("Snapshot list encountered but no header found!")
                 raise ValueError(msg)
-            del lines_after[0]
             real_details = []
             # This is the sprintf pattern we will try to match
             # "%-10s%-20s%7s%20s%15s"
@@ -119,6 +118,7 @@ class QemuImgInfo(object):
                 date_pieces = line_pieces[5].split(":")
                 if len(date_pieces) != 3:
                     break
+                lines_after.pop(0)
                 real_details.append({
                     'id': line_pieces[0],
                     'tag': line_pieces[1],
@@ -126,7 +126,6 @@ class QemuImgInfo(object):
                     'date': line_pieces[3],
                     'vm_clock': line_pieces[4] + " " + line_pieces[5],
                 })
-                del lines_after[0]
         return real_details
 
     def _parse(self, cmd_output):
diff --git a/nova/openstack/common/log.py b/nova/openstack/common/log.py
index 25e437bc0433..e6330c85cdbb 100644
--- a/nova/openstack/common/log.py
+++ b/nova/openstack/common/log.py
@@ -27,6 +27,7 @@ It also allows setting of formatting information through conf.
 
 """
 
+import copy
 import inspect
 import itertools
 import logging
@@ -174,6 +175,16 @@ CONF.register_cli_opts(logging_cli_opts)
 CONF.register_opts(generic_log_opts)
 CONF.register_opts(log_opts)
 
+
+def list_opts():
+    """Entry point for oslo.config-generator."""
+    return [(None, copy.deepcopy(common_cli_opts)),
+            (None, copy.deepcopy(logging_cli_opts)),
+            (None, copy.deepcopy(generic_log_opts)),
+            (None, copy.deepcopy(log_opts)),
+            ]
+
+
 # our new audit level
 # NOTE(jkoelker) Since we synthesized an audit level, make the logging
 #                module aware of it so it acts like other levels.
@@ -498,14 +509,9 @@ def _setup_logging_from_conf(project, version):
         log_root.addHandler(streamlog)
 
     if CONF.publish_errors:
-        try:
-            handler = importutils.import_object(
-                "nova.openstack.common.log_handler.PublishErrorsHandler",
-                logging.ERROR)
-        except ImportError:
-            handler = importutils.import_object(
-                "oslo.messaging.notify.log_handler.PublishErrorsHandler",
-                logging.ERROR)
+        handler = importutils.import_object(
+            "oslo.messaging.notify.log_handler.PublishErrorsHandler",
+            logging.ERROR)
         log_root.addHandler(handler)
 
     datefmt = CONF.log_date_format
diff --git a/nova/openstack/common/memorycache.py b/nova/openstack/common/memorycache.py
index 4826865a2043..f793c9371661 100644
--- a/nova/openstack/common/memorycache.py
+++ b/nova/openstack/common/memorycache.py
@@ -16,6 +16,8 @@
 
 """Super simple fake memcache client."""
 
+import copy
+
 from oslo.config import cfg
 from oslo.utils import timeutils
 
@@ -28,6 +30,11 @@ CONF = cfg.CONF
 CONF.register_opts(memcache_opts)
 
 
+def list_opts():
+    """Entry point for oslo.config-generator."""
+    return [(None, copy.deepcopy(memcache_opts))]
+
+
 def get_client(memcached_servers=None):
     client_cls = Client
 
diff --git a/nova/openstack/common/periodic_task.py b/nova/openstack/common/periodic_task.py
index 2e32d0594e06..679a1d9e0ac8 100644
--- a/nova/openstack/common/periodic_task.py
+++ b/nova/openstack/common/periodic_task.py
@@ -11,6 +11,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import copy
 import random
 import time
 
@@ -36,6 +37,11 @@ LOG = logging.getLogger(__name__)
 DEFAULT_INTERVAL = 60.0
 
 
+def list_opts():
+    """Entry point for oslo.config-generator."""
+    return [(None, copy.deepcopy(periodic_opts))]
+
+
 class InvalidPeriodicTaskArg(Exception):
     message = _("Unexpected argument for periodic task creation: %(arg)s.")
 
diff --git a/nova/openstack/common/policy.py b/nova/openstack/common/policy.py
index 4d7628163790..fe45e5a13f4b 100644
--- a/nova/openstack/common/policy.py
+++ b/nova/openstack/common/policy.py
@@ -77,6 +77,7 @@ as it allows particular rules to be explicitly disabled.
 
 import abc
 import ast
+import copy
 import os
 import re
 
@@ -101,8 +102,8 @@ policy_opts = [
                       'found.')),
     cfg.MultiStrOpt('policy_dirs',
                     default=['policy.d'],
-                    help=_('The directories of policy configuration files is '
-                           'stored')),
+                    help=_('Directories where policy configuration files are '
+                           'stored.')),
 ]
 
 CONF = cfg.CONF
@@ -113,6 +114,11 @@ LOG = logging.getLogger(__name__)
 _checks = {}
 
 
+def list_opts():
+    """Entry point for oslo.config-generator."""
+    return [(None, copy.deepcopy(policy_opts))]
+
+
 class PolicyNotAuthorized(Exception):
 
     def __init__(self, rule):
@@ -193,8 +199,8 @@ class Enforcer(object):
 
     def __init__(self, policy_file=None, rules=None,
                  default_rule=None, use_conf=True):
-        self.rules = Rules(rules, default_rule)
         self.default_rule = default_rule or CONF.policy_default_rule
+        self.rules = Rules(rules, self.default_rule)
 
         self.policy_path = None
         self.policy_file = policy_file or CONF.policy_file
@@ -245,7 +251,7 @@ class Enforcer(object):
                 try:
                     path = self._get_policy_path(path)
                 except cfg.ConfigFilesNotFoundError:
-                    LOG.warn(_LW("Can not find policy directories %s"), path)
+                    LOG.warn(_LW("Can not find policy directory: %s"), path)
                     continue
                 self._walk_through_policy_directory(path,
                                                     self._load_policy_file,
@@ -299,7 +305,7 @@ class Enforcer(object):
         :param do_raise: Whether to raise an exception or not if check
                         fails.
         :param exc: Class of the exception to raise if the check fails.
-                    Any remaining arguments passed to check() (both
+                    Any remaining arguments passed to enforce() (both
                     positional and keyword arguments) will be passed to
                     the exception class. If not specified, PolicyNotAuthorized
                     will be used.
diff --git a/nova/openstack/common/report/generators/threading.py b/nova/openstack/common/report/generators/threading.py
index 5fa081538635..26bbef818e23 100644
--- a/nova/openstack/common/report/generators/threading.py
+++ b/nova/openstack/common/report/generators/threading.py
@@ -44,8 +44,7 @@ class ThreadReportGenerator(object):
             for thread_id, stack in sys._current_frames().items()
         ]
 
-        thread_pairs = dict(zip(range(len(threadModels)), threadModels))
-        return mwdv.ModelWithDefaultViews(thread_pairs,
+        return mwdv.ModelWithDefaultViews(threadModels,
                                           text_view=text_views.MultiView())
 
 
@@ -68,6 +67,5 @@ class GreenThreadReportGenerator(object):
             for gr in rutils._find_objects(greenlet.greenlet)
         ]
 
-        thread_pairs = dict(zip(range(len(threadModels)), threadModels))
-        return mwdv.ModelWithDefaultViews(thread_pairs,
+        return mwdv.ModelWithDefaultViews(threadModels,
                                           text_view=text_views.MultiView())
diff --git a/nova/openstack/common/report/models/base.py b/nova/openstack/common/report/models/base.py
index 7c5d7b32a343..a9b070b83f3d 100644
--- a/nova/openstack/common/report/models/base.py
+++ b/nova/openstack/common/report/models/base.py
@@ -37,13 +37,29 @@ class ReportModel(col.MutableMapping):
     model.  An appropriate object for a view is callable with
     a single parameter: the model to be serialized.
 
-    :param data: a dictionary of data to initially associate with the model
+    If present, the object passed in as data will be transformed
+    into a standard python dict.  For mappings, this is fairly
+    straightforward.  For sequences, the indices become keys
+    and the items become values.
+
+    :param data: a sequence or mapping of data to associate with the model
     :param attached_view: a view object to attach to this model
     """
 
     def __init__(self, data=None, attached_view=None):
         self.attached_view = attached_view
-        self.data = data or {}
+
+        if data is not None:
+            if isinstance(data, col.Mapping):
+                self.data = dict(data)
+            elif isinstance(data, col.Sequence):
+                # convert a list [a, b, c] to a dict {0: a, 1: b, 2: c}
+                self.data = dict(enumerate(data))
+            else:
+                raise TypeError('Data for the model must be a sequence '
+                                'or mapping.')
+        else:
+            self.data = {}
 
     def __str__(self):
         self_cpy = copy.deepcopy(self)
@@ -89,6 +105,8 @@ class ReportModel(col.MutableMapping):
         try:
             return self.data[attrname]
         except KeyError:
+            # we don't have that key in data, and the
+            # model class doesn't have that attribute
             raise AttributeError(
                 "'{cl}' object has no attribute '{an}'".format(
                     cl=type(self).__name__, an=attrname
diff --git a/nova/openstack/common/report/models/conf.py b/nova/openstack/common/report/models/conf.py
index 3c886493c64d..8a0ef3796bc3 100644
--- a/nova/openstack/common/report/models/conf.py
+++ b/nova/openstack/common/report/models/conf.py
@@ -41,8 +41,15 @@ class ConfigModel(mwdv.ModelWithDefaultViews):
         def opt_title(optname, co):
             return co._opts[optname]['opt'].name
 
+        def opt_value(opt_obj, value):
+            if opt_obj['opt'].secret:
+                return '*******'
+            else:
+                return value
+
         self['default'] = dict(
-            (opt_title(optname, conf_obj), conf_obj[optname])
+            (opt_title(optname, conf_obj),
+             opt_value(conf_obj._opts[optname], conf_obj[optname]))
             for optname in conf_obj._opts
         )
 
@@ -50,9 +57,10 @@ class ConfigModel(mwdv.ModelWithDefaultViews):
         for groupname in conf_obj._groups:
             group_obj = conf_obj._groups[groupname]
             curr_group_opts = dict(
-                (opt_title(optname, group_obj), conf_obj[groupname][optname])
-                for optname in group_obj._opts
-            )
+                (opt_title(optname, group_obj),
+                 opt_value(group_obj._opts[optname],
+                           conf_obj[groupname][optname]))
+                for optname in group_obj._opts)
             groups[group_obj.name] = curr_group_opts
 
         self.update(groups)
diff --git a/nova/openstack/common/sslutils.py b/nova/openstack/common/sslutils.py
index 2643f76d0b86..69b6fef80ce8 100644
--- a/nova/openstack/common/sslutils.py
+++ b/nova/openstack/common/sslutils.py
@@ -12,6 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import copy
 import os
 import ssl
 
@@ -32,9 +33,14 @@ ssl_opts = [
                     "the server securely."),
 ]
 
-
 CONF = cfg.CONF
-CONF.register_opts(ssl_opts, "ssl")
+config_section = 'ssl'
+CONF.register_opts(ssl_opts, config_section)
+
+
+def list_opts():
+    """Entry point for oslo.config-generator."""
+    return [(config_section, copy.deepcopy(ssl_opts))]
 
 
 def is_enabled():
diff --git a/nova/openstack/common/threadgroup.py b/nova/openstack/common/threadgroup.py
index ef45b5e33437..b93500bc7c07 100644
--- a/nova/openstack/common/threadgroup.py
+++ b/nova/openstack/common/threadgroup.py
@@ -96,6 +96,8 @@ class ThreadGroup(object):
                 continue
             try:
                 x.stop()
+            except eventlet.greenlet.GreenletExit:
+                pass
             except Exception as ex:
                 LOG.exception(ex)
 
diff --git a/openstack-common.conf b/openstack-common.conf
index a52f990b9a05..86aaec39c162 100644
--- a/openstack-common.conf
+++ b/openstack-common.conf
@@ -2,22 +2,15 @@
 
 # The list of modules to copy from oslo-incubator
 module=cliutils
-module=config
 module=context
 module=eventlet_backdoor
 module=fileutils
-module=fixture.config
 module=fixture.logging
-module=fixture.mockpatch
-module=fixture.moxstubout
 module=imageutils
-module=install_venv_common
 module=local
 module=log
 module=loopingcall
 module=memorycache
-module=middleware.base
-module=middleware.request_id
 module=periodic_task
 module=policy
 module=report