Merge "Create OpenStackClients convenience class"

devstack/lib/watcher

@@ -125,14 +125,8 @@ function create_watcher_conf {
     iniset $WATCHER_CONF oslo_messaging_rabbit rabbit_password $RABBIT_PASSWORD
     iniset $WATCHER_CONF oslo_messaging_rabbit rabbit_host $RABBIT_HOST
 
-    iniset $WATCHER_CONF keystone_authtoken admin_user watcher
-    iniset $WATCHER_CONF keystone_authtoken admin_password $SERVICE_PASSWORD
-    iniset $WATCHER_CONF keystone_authtoken admin_tenant_name $SERVICE_TENANT_NAME
-
     configure_auth_token_middleware $WATCHER_CONF watcher $WATCHER_AUTH_CACHE_DIR
-
-    iniset $WATCHER_CONF keystone_authtoken auth_uri $KEYSTONE_SERVICE_URI/v3
-    iniset $WATCHER_CONF keystone_authtoken auth_version v3
+    configure_auth_token_middleware $WATCHER_CONF watcher $WATCHER_AUTH_CACHE_DIR "watcher_clients_auth"
 
     if is_fedora || is_suse; then
         # watcher defaults to /usr/local/bin, but fedora and suse pip like to

doc/source/deploy/configuration.rst

@@ -166,11 +166,17 @@ The configuration file is organized into the following sections:
 * ``[api]`` - API server configuration
 * ``[database]`` - SQL driver configuration
 * ``[keystone_authtoken]`` - Keystone Authentication plugin configuration
+* ``[watcher_clients_auth]`` - Keystone auth configuration for clients
 * ``[watcher_applier]`` - Watcher Applier module configuration
 * ``[watcher_decision_engine]`` - Watcher Decision Engine module configuration
 * ``[watcher_goals]`` - Goals mapping configuration
 * ``[watcher_strategies]`` - Strategy configuration
 * ``[oslo_messaging_rabbit]`` - Oslo Messaging RabbitMQ driver configuration
+* ``[ceilometer_client]`` - Ceilometer client configuration
+* ``[cinder_client]`` - Cinder client configuration
+* ``[glance_client]`` - Glance client configuration
+* ``[nova_client]`` - Nova client configuration
+* ``[neutron_client]`` - Neutron client configuration
 
 The Watcher configuration file is expected to be named
 ``watcher.conf``. When starting Watcher, you can specify a different
@@ -246,7 +252,11 @@ so that the watcher service is configured for your needs.
 
     # Complete public Identity API endpoint (string value)
     #auth_uri=<None>
-    auth_uri=http://IDENTITY_IP:5000/v3
+    auth_uri=http://IDENTITY_IP:5000/
+
+    # API version of the admin Identity API endpoint. (string value)
+    #auth_version=<None>
+    auth_version=v3
 
     # Complete admin Identity API endpoint. This should specify the
     # unversioned root endpoint e.g. https://localhost:35357/ (string
@@ -271,6 +281,46 @@ so that the watcher service is configured for your needs.
     # value)
     #signing_dir=<None>
 
+#. Configure the credentials to use to authenticate with the Identity Service
+   for the different project clients::
+
+    [watcher_clients_auth]
+
+    # Authentication type to load (unknown value)
+    # Deprecated group/name - [DEFAULT]/auth_plugin
+    #auth_type = <None>
+    auth_type = password
+
+    # Authentication URL (unknown value)
+    #auth_url = <None>
+    auth_url = http://IDENTITY_IP:35357
+
+    # Username (unknown value)
+    # Deprecated group/name - [DEFAULT]/username
+    #username = <None>
+    username=watcher
+
+    # User's password (unknown value)
+    #password = <None>
+    password = WATCHER_PASSWORD
+
+    # Domain ID containing project (unknown value)
+    #project_domain_id = <None>
+    project_domain_id = default
+
+    # User's domain id (unknown value)
+    #user_domain_id = <None>
+    user_domain_id = default
+
+#. Configure the clients to use a specific version if desired. For example, to
+   configure Watcher to use a Nova client with version 2.1, use::
+
+    [nova_client]
+
+    # Version of Nova API to use in novaclient. (string value)
+    #api_version = 2
+    api_version = 2.1
+
 #. Create the Watcher Service database tables::
 
     $ watcher-db-manage --config-file /etc/watcher/watcher.conf create_schema

requirements.txt

@@ -4,6 +4,7 @@
 
 enum34;python_version=='2.7' or python_version=='2.6'
 jsonpatch>=1.1
+keystoneauth1>=2.1.0
 keystonemiddleware>=2.0.0,!=2.4.0
 oslo.config>=2.3.0  # Apache-2.0
 oslo.db>=2.4.1      # Apache-2.0

watcher/applier/action_plan/default.py

@@ -52,7 +52,7 @@ class DefaultActionPlanHandler(base.BaseActionPlanHandler):
             self.notify(self.action_plan_uuid,
                         event_types.EventTypes.LAUNCH_ACTION_PLAN,
                         ap_objects.State.ONGOING)
-            applier = default.DefaultApplier(self.applier_manager, self.ctx)
+            applier = default.DefaultApplier(self.ctx, self.applier_manager)
             result = applier.execute(self.action_plan_uuid)
         except Exception as e:
             LOG.exception(e)

watcher/applier/actions/base.py

@@ -22,12 +22,22 @@ import abc
 
 import six
 
+from watcher.common import clients
+
 
 @six.add_metaclass(abc.ABCMeta)
 class BaseAction(object):
-    def __init__(self):
+    def __init__(self, osc=None):
+        """:param osc: an OpenStackClients instance"""
         self._input_parameters = {}
         self._applies_to = ""
+        self._osc = osc
+
+    @property
+    def osc(self):
+        if not self._osc:
+            self._osc = clients.OpenStackClients()
+        return self._osc
 
     @property
     def input_parameters(self):

watcher/applier/actions/change_nova_service_state.py

@@ -21,8 +21,7 @@
 from watcher._i18n import _
 from watcher.applier.actions import base
 from watcher.common import exception
-from watcher.common import keystone as kclient
-from watcher.common import nova as nclient
+from watcher.common import nova_helper
 from watcher.decision_engine.model import hypervisor_state as hstate
 
 
@@ -57,13 +56,11 @@ class ChangeNovaServiceState(base.BaseAction):
             raise exception.IllegalArgumentException(
                 message=_("The target state is not defined"))
 
-        keystone = kclient.KeystoneClient()
-        wrapper = nclient.NovaClient(keystone.get_credentials(),
-                                     session=keystone.get_session())
+        nova = nova_helper.NovaHelper(osc=self.osc)
         if state is True:
-            return wrapper.enable_service_nova_compute(self.host)
+            return nova.enable_service_nova_compute(self.host)
         else:
-            return wrapper.disable_service_nova_compute(self.host)
+            return nova.disable_service_nova_compute(self.host)
 
     def precondition(self):
         pass

watcher/applier/actions/factory.py

@@ -28,9 +28,10 @@ class ActionFactory(object):
     def __init__(self):
         self.action_loader = default.DefaultActionLoader()
 
-    def make_action(self, object_action):
+    def make_action(self, object_action, osc=None):
         LOG.debug("Creating instance of %s", object_action.action_type)
-        loaded_action = self.action_loader.load(name=object_action.action_type)
+        loaded_action = self.action_loader.load(name=object_action.action_type,
+                                                osc=osc)
         loaded_action.input_parameters = object_action.input_parameters
         loaded_action.applies_to = object_action.applies_to
         return loaded_action

watcher/applier/actions/migration.py

@@ -21,8 +21,7 @@ from oslo_log import log
 
 from watcher.applier.actions import base
 from watcher.common import exception
-from watcher.common import keystone as kclient
-from watcher.common import nova as nclient
+from watcher.common import nova_helper
 
 LOG = log.getLogger(__name__)
 
@@ -45,15 +44,13 @@ class Migrate(base.BaseAction):
         return self.input_parameters.get('src_hypervisor')
 
     def migrate(self, destination):
-        keystone = kclient.KeystoneClient()
-        wrapper = nclient.NovaClient(keystone.get_credentials(),
-                                     session=keystone.get_session())
-        LOG.debug("Migrate instance %s to %s  ", self.instance_uuid,
+        nova = nova_helper.NovaHelper(osc=self.osc)
+        LOG.debug("Migrate instance %s to %s", self.instance_uuid,
                   destination)
-        instance = wrapper.find_instance(self.instance_uuid)
+        instance = nova.find_instance(self.instance_uuid)
         if instance:
             if self.migration_type == 'live':
-                return wrapper.live_migrate_instance(
+                return nova.live_migrate_instance(
                     instance_id=self.instance_uuid, dest_hostname=destination)
             else:
                 raise exception.Invalid(

watcher/applier/default.py

@@ -28,7 +28,7 @@ CONF = cfg.CONF
 
 
 class DefaultApplier(base.BaseApplier):
-    def __init__(self, applier_manager, context):
+    def __init__(self, context, applier_manager):
         super(DefaultApplier, self).__init__()
         self._applier_manager = applier_manager
         self._loader = default.DefaultWorkFlowEngineLoader()
@@ -48,9 +48,10 @@ class DefaultApplier(base.BaseApplier):
         if self._engine is None:
             selected_workflow_engine = CONF.watcher_applier.workflow_engine
             LOG.debug("Loading workflow engine %s ", selected_workflow_engine)
-            self._engine = self._loader.load(name=selected_workflow_engine)
-            self._engine.context = self.context
-            self._engine.applier_manager = self.applier_manager
+            self._engine = self._loader.load(
+                name=selected_workflow_engine,
+                context=self.context,
+                applier_manager=self.applier_manager)
         return self._engine
 
     def execute(self, action_plan_uuid):

watcher/applier/workflow_engine/base.py

@@ -22,33 +22,33 @@ import six
 
 from watcher.applier.actions import factory
 from watcher.applier.messaging import event_types
+from watcher.common import clients
 from watcher.common.messaging.events import event
 from watcher import objects
 
 
 @six.add_metaclass(abc.ABCMeta)
 class BaseWorkFlowEngine(object):
-    def __init__(self):
-        self._applier_manager = None
-        self._context = None
+    def __init__(self, context=None, applier_manager=None):
+        self._context = context
+        self._applier_manager = applier_manager
         self._action_factory = factory.ActionFactory()
+        self._osc = None
 
     @property
     def context(self):
         return self._context
 
-    @context.setter
-    def context(self, c):
-        self._context = c
+    @property
+    def osc(self):
+        if not self._osc:
+            self._osc = clients.OpenStackClients()
+        return self._osc
 
     @property
     def applier_manager(self):
         return self._applier_manager
 
-    @applier_manager.setter
-    def applier_manager(self, a):
-        self._applier_manager = a
-
     @property
     def action_factory(self):
         return self._action_factory

watcher/applier/workflow_engine/default.py

@@ -89,7 +89,9 @@ class TaskFlowActionContainer(task.Task):
     @property
     def action(self):
         if self.loaded_action is None:
-            action = self.engine.action_factory.make_action(self._db_action)
+            action = self.engine.action_factory.make_action(
+                self._db_action,
+                osc=self._engine.osc)
             self.loaded_action = action
         return self.loaded_action
 

watcher/common/ceilometer_helper.py

@@ -17,30 +17,16 @@
 # limitations under the License.
 #
 
-from ceilometerclient import client
 from ceilometerclient.exc import HTTPUnauthorized
 
-from watcher.common import keystone
+from watcher.common import clients
 
 
-class CeilometerClient(object):
-    def __init__(self, api_version='2'):
-        self._cmclient = None
-        self._api_version = api_version
-
-    @property
-    def cmclient(self):
-        """Initialization of Ceilometer client."""
-        if not self._cmclient:
-            ksclient = keystone.KeystoneClient()
-            creds = ksclient.get_credentials()
-            endpoint = ksclient.get_endpoint(
-                service_type='metering',
-                endpoint_type='publicURL')
-            self._cmclient = client.get_client(self._api_version,
-                                               ceilometer_url=endpoint,
-                                               **creds)
-        return self._cmclient
+class CeilometerHelper(object):
+    def __init__(self, osc=None):
+        """:param osc: an OpenStackClients instance"""
+        self.osc = osc if osc else clients.OpenStackClients()
+        self.ceilometer = self.osc.ceilometer()
 
     def build_query(self, user_id=None, tenant_id=None, resource_id=None,
                     user_ids=None, tenant_ids=None, resource_ids=None):
@@ -83,20 +69,21 @@ class CeilometerClient(object):
         try:
             return f(*args, **kargs)
         except HTTPUnauthorized:
-            self.reset_client()
+            self.osc.reset_clients()
+            self.ceilometer = self.osc.ceilometer()
             return f(*args, **kargs)
         except Exception:
             raise
 
     def query_sample(self, meter_name, query, limit=1):
-        return self.query_retry(f=self.cmclient.samples.list,
+        return self.query_retry(f=self.ceilometer.samples.list,
                                 meter_name=meter_name,
                                 limit=limit,
                                 q=query)
 
     def statistic_list(self, meter_name, query=None, period=None):
         """List of statistics."""
-        statistics = self.cmclient.statistics.list(
+        statistics = self.ceilometer.statistics.list(
             meter_name=meter_name,
             q=query,
             period=period)
@@ -104,7 +91,8 @@ class CeilometerClient(object):
 
     def meter_list(self, query=None):
         """List the user's meters."""
-        meters = self.query_retry(f=self.cmclient.meters.list, query=query)
+        meters = self.query_retry(f=self.ceilometer.meters.list,
+                                  query=query)
         return meters
 
     def statistic_aggregation(self,
@@ -125,7 +113,7 @@ class CeilometerClient(object):
         """
 
         query = self.build_query(resource_id=resource_id)
-        statistic = self.query_retry(f=self.cmclient.statistics.list,
+        statistic = self.query_retry(f=self.ceilometer.statistics.list,
                                      meter_name=meter_name,
                                      q=query,
                                      period=period,
@@ -156,6 +144,3 @@ class CeilometerClient(object):
             return samples[-1]._info['counter_volume']
         else:
             return False
-
-    def reset_client(self):
-        self._cmclient = None

watcher/common/clients.py

@@ -0,0 +1,158 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from ceilometerclient import client as ceclient
+from cinderclient import client as ciclient
+from glanceclient import client as glclient
+from keystoneauth1 import loading as ka_loading
+from keystoneclient import client as keyclient
+from neutronclient.neutron import client as netclient
+from novaclient import client as nvclient
+from oslo_config import cfg
+
+from watcher._i18n import _
+from watcher.common import exception
+
+
+NOVA_CLIENT_OPTS = [
+    cfg.StrOpt('api_version',
+               default='2',
+               help=_('Version of Nova API to use in novaclient.'))]
+
+GLANCE_CLIENT_OPTS = [
+    cfg.StrOpt('api_version',
+               default='2',
+               help=_('Version of Glance API to use in glanceclient.'))]
+
+CINDER_CLIENT_OPTS = [
+    cfg.StrOpt('api_version',
+               default='2',
+               help=_('Version of Cinder API to use in cinderclient.'))]
+
+CEILOMETER_CLIENT_OPTS = [
+    cfg.StrOpt('api_version',
+               default='2',
+               help=_('Version of Ceilometer API to use in '
+                      'ceilometerclient.'))]
+
+NEUTRON_CLIENT_OPTS = [
+    cfg.StrOpt('api_version',
+               default='2',
+               help=_('Version of Neutron API to use in neutronclient.'))]
+
+cfg.CONF.register_opts(NOVA_CLIENT_OPTS, group='nova_client')
+cfg.CONF.register_opts(GLANCE_CLIENT_OPTS, group='glance_client')
+cfg.CONF.register_opts(CINDER_CLIENT_OPTS, group='cinder_client')
+cfg.CONF.register_opts(CEILOMETER_CLIENT_OPTS, group='ceilometer_client')
+cfg.CONF.register_opts(NEUTRON_CLIENT_OPTS, group='neutron_client')
+
+_CLIENTS_AUTH_GROUP = 'watcher_clients_auth'
+
+ka_loading.register_auth_conf_options(cfg.CONF, _CLIENTS_AUTH_GROUP)
+ka_loading.register_session_conf_options(cfg.CONF, _CLIENTS_AUTH_GROUP)
+
+
+class OpenStackClients(object):
+    """Convenience class to create and cache client instances."""
+
+    def __init__(self):
+        self.reset_clients()
+
+    def reset_clients(self):
+        self._session = None
+        self._keystone = None
+        self._nova = None
+        self._glance = None
+        self._cinder = None
+        self._ceilometer = None
+        self._neutron = None
+
+    def _get_keystone_session(self):
+        auth = ka_loading.load_auth_from_conf_options(cfg.CONF,
+                                                      _CLIENTS_AUTH_GROUP)
+        sess = ka_loading.load_session_from_conf_options(cfg.CONF,
+                                                         _CLIENTS_AUTH_GROUP,
+                                                         auth=auth)
+        return sess
+
+    @property
+    def auth_url(self):
+        return self.keystone().auth_url
+
+    @property
+    def session(self):
+        if not self._session:
+            self._session = self._get_keystone_session()
+        return self._session
+
+    def _get_client_option(self, client, option):
+        return getattr(getattr(cfg.CONF, '%s_client' % client), option)
+
+    @exception.wrap_keystone_exception
+    def keystone(self):
+        if not self._keystone:
+            self._keystone = keyclient.Client(session=self.session)
+
+        return self._keystone
+
+    @exception.wrap_keystone_exception
+    def nova(self):
+        if self._nova:
+            return self._nova
+
+        novaclient_version = self._get_client_option('nova', 'api_version')
+        self._nova = nvclient.Client(novaclient_version,
+                                     session=self.session)
+        return self._nova
+
+    @exception.wrap_keystone_exception
+    def glance(self):
+        if self._glance:
+            return self._glance
+
+        glanceclient_version = self._get_client_option('glance', 'api_version')
+        self._glance = glclient.Client(glanceclient_version,
+                                       session=self.session)
+        return self._glance
+
+    @exception.wrap_keystone_exception
+    def cinder(self):
+        if self._cinder:
+            return self._cinder
+
+        cinderclient_version = self._get_client_option('cinder', 'api_version')
+        self._cinder = ciclient.Client(cinderclient_version,
+                                       session=self.session)
+        return self._cinder
+
+    @exception.wrap_keystone_exception
+    def ceilometer(self):
+        if self._ceilometer:
+            return self._ceilometer
+
+        ceilometerclient_version = self._get_client_option('ceilometer',
+                                                           'api_version')
+        self._ceilometer = ceclient.Client(ceilometerclient_version,
+                                           session=self.session)
+        return self._ceilometer
+
+    @exception.wrap_keystone_exception
+    def neutron(self):
+        if self._neutron:
+            return self._neutron
+
+        neutronclient_version = self._get_client_option('neutron',
+                                                        'api_version')
+        self._neutron = netclient.Client(neutronclient_version,
+                                         session=self.session)
+        self._neutron.format = 'json'
+        return self._neutron

watcher/common/exception.py

@@ -22,6 +22,10 @@ SHOULD include dedicated exception logging.
 
 """
 
+import functools
+import sys
+
+from keystoneclient import exceptions as keystone_exceptions
 from oslo_config import cfg
 from oslo_log import log as logging
 import six
@@ -40,6 +44,23 @@ CONF = cfg.CONF
 CONF.register_opts(exc_log_opts)
 
 
+def wrap_keystone_exception(func):
+    """Wrap keystone exceptions and throw Watcher specific exceptions."""
+    @functools.wraps(func)
+    def wrapped(*args, **kw):
+        try:
+            return func(*args, **kw)
+        except keystone_exceptions.AuthorizationFailure:
+            raise AuthorizationFailure(
+                client=func.__name__, reason=sys.exc_info()[1])
+        except keystone_exceptions.ClientException:
+            raise AuthorizationFailure(
+                client=func.__name__,
+                reason=(_('Unexpected keystone client error occurred: %s')
+                        % sys.exc_info()[1]))
+    return wrapped
+
+
 class WatcherException(Exception):
     """Base Watcher Exception
 
@@ -226,6 +247,10 @@ class NoDataFound(WatcherException):
     msg_fmt = _('No rows were returned')
 
 
+class AuthorizationFailure(WatcherException):
+    msg_fmt = _('%(client)s connection failed. Reason: %(reason)s')
+
+
 class KeystoneFailure(WatcherException):
     msg_fmt = _("'Keystone API endpoint is missing''")
 

watcher/common/keystone.py

@@ -1,131 +0,0 @@
-# -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
-#
-# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from keystoneclient.auth.identity import generic
-from keystoneclient import session as keystone_session
-from oslo_config import cfg
-from oslo_log import log
-from six.moves.urllib.parse import urljoin
-from six.moves.urllib.parse import urlparse
-
-from watcher._i18n import _
-from watcher.common import exception
-
-
-LOG = log.getLogger(__name__)
-CONF = cfg.CONF
-
-CONF.import_opt('admin_user', 'keystonemiddleware.auth_token',
-                group='keystone_authtoken')
-CONF.import_opt('admin_tenant_name', 'keystonemiddleware.auth_token',
-                group='keystone_authtoken')
-CONF.import_opt('admin_password', 'keystonemiddleware.auth_token',
-                group='keystone_authtoken')
-CONF.import_opt('auth_uri', 'keystonemiddleware.auth_token',
-                group='keystone_authtoken')
-CONF.import_opt('auth_version', 'keystonemiddleware.auth_token',
-                group='keystone_authtoken')
-CONF.import_opt('insecure', 'keystonemiddleware.auth_token',
-                group='keystone_authtoken')
-
-
-class KeystoneClient(object):
-    def __init__(self):
-        self._ks_client = None
-        self._session = None
-        self._auth = None
-        self._token = None
-
-    def get_endpoint(self, **kwargs):
-        kc = self.get_ksclient()
-        if not kc.has_service_catalog():
-            raise exception.KeystoneFailure(
-                _('No Keystone service catalog loaded')
-            )
-        attr = None
-        filter_value = None
-        if kwargs.get('region_name'):
-            attr = 'region'
-            filter_value = kwargs.get('region_name')
-        return kc.service_catalog.url_for(
-            service_type=kwargs.get('service_type') or 'metering',
-            attr=attr,
-            filter_value=filter_value,
-            endpoint_type=kwargs.get('endpoint_type') or 'publicURL')
-
-    def _is_apiv3(self, auth_url, auth_version):
-        return auth_version == 'v3.0' or '/v3' in urlparse(auth_url).path
-
-    def get_keystone_url(self, auth_url, auth_version):
-        """Gives an http/https url to contact keystone."""
-
-        api_v3 = self._is_apiv3(auth_url, auth_version)
-        api_version = 'v3' if api_v3 else 'v2.0'
-        # NOTE(lucasagomes): Get rid of the trailing '/' otherwise urljoin()
-        #   fails to override the version in the URL
-        return urljoin(auth_url.rstrip('/'), api_version)
-
-    def get_ksclient(self, creds=None):
-        """Get an endpoint and auth token from Keystone."""
-        auth_version = CONF.keystone_authtoken.auth_version
-        auth_url = CONF.keystone_authtoken.auth_uri
-        api_v3 = self._is_apiv3(auth_url, auth_version)
-        if creds is None:
-            ks_args = self._get_credentials(api_v3)
-        else:
-            ks_args = creds
-
-        if api_v3:
-            from keystoneclient.v3 import client
-        else:
-            from keystoneclient.v2_0 import client
-        # generic
-        # ksclient = client.Client(version=api_version,
-        # session=session,**ks_args)
-
-        return client.Client(**ks_args)
-
-    def _get_credentials(self, api_v3):
-        if api_v3:
-            creds = \
-                {'auth_url': CONF.keystone_authtoken.auth_uri,
-                 'username': CONF.keystone_authtoken.admin_user,
-                 'password': CONF.keystone_authtoken.admin_password,
-                 'project_name': CONF.keystone_authtoken.admin_tenant_name,
-                 'user_domain_name': "default",
-                 'project_domain_name': "default"}
-        else:
-            creds = \
-                {'auth_url': CONF.keystone_authtoken.auth_uri,
-                 'username': CONF.keystone_authtoken.admin_user,
-                 'password': CONF.keystone_authtoken.admin_password,
-                 'tenant_name': CONF.keystone_authtoken.admin_tenant_name}
-        LOG.debug(creds)
-        return creds
-
-    def get_credentials(self):
-        api_v3 = self._is_apiv3(CONF.keystone_authtoken.auth_uri,
-                                CONF.keystone_authtoken.auth_version)
-        return self._get_credentials(api_v3)
-
-    def get_session(self):
-        creds = self.get_credentials()
-        self._auth = generic.Password(**creds)
-        session = keystone_session.Session(auth=self._auth)
-        return session

watcher/common/loader/default.py

@@ -31,7 +31,7 @@ class DefaultLoader(BaseLoader):
         super(DefaultLoader, self).__init__()
         self.namespace = namespace
 
-    def load(self, name):
+    def load(self, name, **kwargs):
         try:
             LOG.debug("Loading in namespace %s => %s ", self.namespace, name)
             driver_manager = DriverManager(namespace=self.namespace,
@@ -41,7 +41,7 @@ class DefaultLoader(BaseLoader):
             LOG.exception(exc)
             raise exception.LoadingError(name=name)
 
-        return loaded()
+        return loaded(**kwargs)
 
     def list_available(self):
         extension_manager = ExtensionManager(namespace=self.namespace)

watcher/common/nova_helper.py

@@ -23,29 +23,22 @@ import time
 from oslo_log import log
 
 import cinderclient.exceptions as ciexceptions
-import cinderclient.v2.client as ciclient
-import glanceclient.v2.client as glclient
-import neutronclient.neutron.client as netclient
-import novaclient.client as nvclient
 import novaclient.exceptions as nvexceptions
 
-from watcher.common import keystone
+from watcher.common import clients
 
 LOG = log.getLogger(__name__)
 
 
-class NovaClient(object):
-    NOVA_CLIENT_API_VERSION = "2"
+class NovaHelper(object):
 
-    def __init__(self, creds, session):
-        self.user = creds['username']
-        self.session = session
-        self.neutron = None
-        self.cinder = None
-        self.nova = nvclient.Client(self.NOVA_CLIENT_API_VERSION,
-                                    session=session)
-        self.keystone = keystone.KeystoneClient().get_ksclient(creds)
-        self.glance = None
+    def __init__(self, osc=None):
+        """:param osc: an OpenStackClients instance"""
+        self.osc = osc if osc else clients.OpenStackClients()
+        self.neutron = self.osc.neutron()
+        self.cinder = self.osc.cinder()
+        self.nova = self.osc.nova()
+        self.glance = self.osc.glance()
 
     def get_hypervisors_list(self):
         return self.nova.hypervisors.list()
@@ -180,9 +173,6 @@ class NovaClient(object):
                 volume_id = attached_volume['id']
 
                 try:
-                    if self.cinder is None:
-                        self.cinder = ciclient.Client('2',
-                                                      session=self.session)
                     volume = self.cinder.volumes.get(volume_id)
 
                     attachments_list = getattr(volume, "attachments")
@@ -446,13 +436,6 @@ class NovaClient(object):
         :param metadata: a dictionary containing the list of
             key-value pairs to associate to the image as metadata.
         """
-        if self.glance is None:
-            glance_endpoint = self.keystone. \
-                service_catalog.url_for(service_type='image',
-                                        endpoint_type='publicURL')
-            self.glance = glclient.Client(glance_endpoint,
-                                          token=self.keystone.auth_token)
-
         LOG.debug(
             "Trying to create an image from instance %s ..." % instance_id)
 
@@ -676,10 +659,6 @@ class NovaClient(object):
 
     def get_network_id_from_name(self, net_name="private"):
         """This method returns the unique id of the provided network name"""
-        if self.neutron is None:
-            self.neutron = netclient.Client('2.0', session=self.session)
-            self.neutron.format = 'json'
-
         networks = self.neutron.list_networks(name=net_name)
 
         # LOG.debug(networks)

watcher/decision_engine/strategy/context/default.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 from oslo_log import log
 
+from watcher.common import clients
 from watcher.decision_engine.strategy.context.base import BaseStrategyContext
 from watcher.decision_engine.strategy.selection.default import \
     DefaultStrategySelector
@@ -47,15 +48,17 @@ class DefaultStrategyContext(BaseStrategyContext):
         audit_template = objects.\
             AuditTemplate.get_by_id(request_context, audit.audit_template_id)
 
+        osc = clients.OpenStackClients()
+
         # todo(jed) retrieve in audit_template parameters (threshold,...)
         # todo(jed) create ActionPlan
-        collector_manager = self.collector.get_cluster_model_collector()
+        collector_manager = self.collector.get_cluster_model_collector(osc=osc)
 
         # todo(jed) remove call to get_latest_cluster_data_model
         cluster_data_model = collector_manager.get_latest_cluster_data_model()
 
-        selected_strategy = self.strategy_selector. \
-            define_from_goal(audit_template.goal)
+        selected_strategy = self.strategy_selector.define_from_goal(
+            audit_template.goal, osc=osc)
 
         # todo(jed) add parameters and remove cluster_data_model
         return selected_strategy.execute(cluster_data_model)

watcher/decision_engine/strategy/selection/default.py

@@ -48,11 +48,12 @@ class DefaultStrategySelector(base.BaseSelector):
         super(DefaultStrategySelector, self).__init__()
         self.strategy_loader = default.DefaultStrategyLoader()
 
-    def define_from_goal(self, goal_name):
+    def define_from_goal(self, goal_name, osc=None):
+        """:param osc: an OpenStackClients instance"""
         strategy_to_load = None
         try:
             strategy_to_load = CONF.watcher_goals.goals[goal_name]
-            return self.strategy_loader.load(strategy_to_load)
+            return self.strategy_loader.load(strategy_to_load, osc=osc)
         except KeyError as exc:
             LOG.exception(exc)
             raise exception.WatcherException(

watcher/decision_engine/strategy/strategies/base.py

@@ -33,10 +33,11 @@ provided as well.
 """
 
 import abc
+
 from oslo_log import log
 import six
 
-
+from watcher.common import clients
 from watcher.decision_engine.solution.default import DefaultSolution
 from watcher.decision_engine.strategy.common.level import StrategyLevel
 
@@ -51,7 +52,8 @@ class BaseStrategy(object):
     Solution for a given Goal.
     """
 
-    def __init__(self, name=None, description=None):
+    def __init__(self, name=None, description=None, osc=None):
+        """:param osc: an OpenStackClients instance"""
         self._name = name
         self.description = description
         # default strategy level
@@ -59,6 +61,7 @@ class BaseStrategy(object):
         self._cluster_state_collector = None
         # the solution given by the strategy
         self._solution = DefaultSolution()
+        self._osc = osc
 
     @abc.abstractmethod
     def execute(self, model):
@@ -70,6 +73,12 @@ class BaseStrategy(object):
         :rtype: :class:`watcher.decision_engine.solution.base.BaseSolution`
         """
 
+    @property
+    def osc(self):
+        if not self._osc:
+            self._osc = clients.OpenStackClients()
+        return self._osc
+
     @property
     def solution(self):
         return self._solution

watcher/decision_engine/strategy/strategies/basic_consolidation.py

@@ -41,7 +41,8 @@ class BasicConsolidation(BaseStrategy):
     MIGRATION = "migrate"
     CHANGE_NOVA_SERVICE_STATE = "change_nova_service_state"
 
-    def __init__(self, name=DEFAULT_NAME, description=DEFAULT_DESCRIPTION):
+    def __init__(self, name=DEFAULT_NAME, description=DEFAULT_DESCRIPTION,
+                 osc=None):
         """Basic offline Consolidation using live migration
 
     The basic consolidation algorithm has several limitations.
@@ -68,8 +69,9 @@ class BasicConsolidation(BaseStrategy):
 
         :param name: the name of the strategy
         :param description: a description of the strategy
+        :param osc: an OpenStackClients object
         """
-        super(BasicConsolidation, self).__init__(name, description)
+        super(BasicConsolidation, self).__init__(name, description, osc)
 
         # set default value for the number of released nodes
         self.number_of_released_nodes = 0
@@ -102,7 +104,7 @@ class BasicConsolidation(BaseStrategy):
     @property
     def ceilometer(self):
         if self._ceilometer is None:
-            self._ceilometer = CeilometerClusterHistory()
+            self._ceilometer = CeilometerClusterHistory(osc=self.osc)
         return self._ceilometer
 
     @ceilometer.setter

watcher/decision_engine/strategy/strategies/dummy_strategy.py

@@ -30,8 +30,9 @@ class DummyStrategy(BaseStrategy):
     NOP = "nop"
     SLEEP = "sleep"
 
-    def __init__(self, name=DEFAULT_NAME, description=DEFAULT_DESCRIPTION):
-        super(DummyStrategy, self).__init__(name, description)
+    def __init__(self, name=DEFAULT_NAME, description=DEFAULT_DESCRIPTION,
+                 osc=None):
+        super(DummyStrategy, self).__init__(name, description, osc)
 
     def execute(self, model):
         parameters = {'message': 'hello World'}

watcher/decision_engine/strategy/strategies/outlet_temp_control.py

@@ -40,7 +40,8 @@ class OutletTempControl(BaseStrategy):
 
     MIGRATION = "migrate"
 
-    def __init__(self, name=DEFAULT_NAME, description=DEFAULT_DESCRIPTION):
+    def __init__(self, name=DEFAULT_NAME, description=DEFAULT_DESCRIPTION,
+                 osc=None):
         """[PoC]Outlet temperature control using live migration
 
         It is a migration strategy based on the Outlet Temperature of physical
@@ -67,8 +68,9 @@ class OutletTempControl(BaseStrategy):
 
         :param name: the name of the strategy
         :param description: a description of the strategy
+        :param osc: an OpenStackClients object
         """
-        super(OutletTempControl, self).__init__(name, description)
+        super(OutletTempControl, self).__init__(name, description, osc)
         # the migration plan will be triggered when the outlet temperature
         # reaches threshold
         # TODO(zhenzanz): Threshold should be configurable for each audit
@@ -79,7 +81,7 @@ class OutletTempControl(BaseStrategy):
     @property
     def ceilometer(self):
         if self._ceilometer is None:
-            self._ceilometer = CeilometerClusterHistory()
+            self._ceilometer = CeilometerClusterHistory(osc=self.osc)
         return self._ceilometer
 
     @ceilometer.setter

watcher/metrics_engine/cluster_history/ceilometer.py

@@ -20,7 +20,7 @@
 
 from oslo_config import cfg
 from oslo_log import log
-from watcher.common.ceilometer import CeilometerClient
+from watcher.common import ceilometer_helper
 
 from watcher.metrics_engine.cluster_history.api import BaseClusterHistory
 
@@ -29,8 +29,10 @@ LOG = log.getLogger(__name__)
 
 
 class CeilometerClusterHistory(BaseClusterHistory):
-    def __init__(self):
-        self.ceilometer = CeilometerClient()
+    def __init__(self, osc=None):
+        """:param osc: an OpenStackClients instance"""
+        super(CeilometerClusterHistory, self).__init__()
+        self.ceilometer = ceilometer_helper.CeilometerHelper(osc=osc)
 
     def statistic_list(self, meter_name, query=None, period=None):
         return self.ceilometer.statistic_list(meter_name, query, period)

watcher/metrics_engine/cluster_model_collector/manager.py

@@ -20,8 +20,7 @@
 from oslo_config import cfg
 from oslo_log import log
 
-from watcher.common.keystone import KeystoneClient
-from watcher.common.nova import NovaClient
+from watcher.common import nova_helper
 from watcher.metrics_engine.cluster_model_collector.nova import \
     NovaClusterModelCollector
 
@@ -30,8 +29,7 @@ CONF = cfg.CONF
 
 
 class CollectorManager(object):
-    def get_cluster_model_collector(self):
-        keystone = KeystoneClient()
-        wrapper = NovaClient(keystone.get_credentials(),
-                             session=keystone.get_session())
-        return NovaClusterModelCollector(wrapper=wrapper)
+    def get_cluster_model_collector(self, osc=None):
+        """:param osc: an OpenStackClients instance"""
+        nova = nova_helper.NovaHelper(osc=osc)
+        return NovaClusterModelCollector(nova)

watcher/opts.py

@@ -15,8 +15,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from keystoneauth1 import loading as ka_loading
+
 import watcher.api.app
 from watcher.applier import manager as applier_manager
+from watcher.common import clients
 from watcher.decision_engine import manager as decision_engine_manger
 from watcher.decision_engine.planner import manager as planner_manager
 from watcher.decision_engine.strategy.selection import default \
@@ -29,7 +32,15 @@ def list_opts():
         ('watcher_goals', strategy_selector.WATCHER_GOALS_OPTS),
         ('watcher_decision_engine',
          decision_engine_manger.WATCHER_DECISION_ENGINE_OPTS),
-        ('watcher_applier',
-         applier_manager.APPLIER_MANAGER_OPTS),
-        ('watcher_planner', planner_manager.WATCHER_PLANNER_OPTS)
+        ('watcher_applier', applier_manager.APPLIER_MANAGER_OPTS),
+        ('watcher_planner', planner_manager.WATCHER_PLANNER_OPTS),
+        ('nova_client', clients.NOVA_CLIENT_OPTS),
+        ('glance_client', clients.GLANCE_CLIENT_OPTS),
+        ('cinder_client', clients.CINDER_CLIENT_OPTS),
+        ('ceilometer_client', clients.CEILOMETER_CLIENT_OPTS),
+        ('neutron_client', clients.NEUTRON_CLIENT_OPTS),
+        ('watcher_clients_auth',
+         (ka_loading.get_auth_common_conf_options() +
+          ka_loading.get_auth_plugin_conf_options('password') +
+          ka_loading.get_session_conf_options()))
     ]

watcher/tests/applier/workflow_engine/test_default_workflow_engine.py

@@ -53,9 +53,9 @@ class FakeAction(abase.BaseAction):
 class TestDefaultWorkFlowEngine(base.DbTestCase):
     def setUp(self):
         super(TestDefaultWorkFlowEngine, self).setUp()
-        self.engine = tflow.DefaultWorkFlowEngine()
-        self.engine.context = self.context
-        self.engine.applier_manager = mock.MagicMock()
+        self.engine = tflow.DefaultWorkFlowEngine(
+            context=self.context,
+            applier_manager=mock.MagicMock())
 
     def test_execute(self):
         actions = mock.MagicMock()

watcher/tests/common/test_ceilometer.py

@@ -1,99 +0,0 @@
-# -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
-from mock import MagicMock
-from mock import mock
-from oslo_config import cfg
-from watcher.common.ceilometer import CeilometerClient
-
-from watcher.tests.base import BaseTestCase
-CONF = cfg.CONF
-
-
-class TestCeilometer(BaseTestCase):
-    def setUp(self):
-        super(TestCeilometer, self).setUp()
-        self.cm = CeilometerClient()
-
-    def test_build_query(self):
-        expected = [{'field': 'user_id', 'op': 'eq', 'value': u'user_id'},
-                    {'field': 'project_id', 'op': 'eq', 'value': u'tenant_id'},
-                    {'field': 'resource_id', 'op': 'eq',
-                     'value': u'resource_id'}]
-
-        query = self.cm.build_query(user_id="user_id",
-                                    tenant_id="tenant_id",
-                                    resource_id="resource_id",
-                                    user_ids=["user_ids"],
-                                    tenant_ids=["tenant_ids"],
-                                    resource_ids=["resource_ids"])
-        self.assertEqual(query, expected)
-
-    @mock.patch('keystoneclient.v2_0.client.Client', autospec=True)
-    @mock.patch('ceilometerclient.v2.client.Client', autospec=True)
-    def test_get_ceilometer_v2(self, mock_keystone, mock_ceilometer):
-        cfg.CONF.set_override(
-            'auth_uri', "http://127.0.0.1:9898/v2", group="keystone_authtoken",
-            enforce_type=True
-        )
-        c = CeilometerClient(api_version='2')
-        from ceilometerclient.v2 import Client
-        self.assertIsInstance(c.cmclient, Client)
-
-    @mock.patch.object(CeilometerClient, "cmclient")
-    def test_statistic_aggregation(self, mock_keystone):
-        statistic = MagicMock()
-        expected_result = 100
-        statistic[-1]._info = {'aggregate': {'avg': expected_result}}
-        mock_keystone.statistics.list.return_value = statistic
-        val = self.cm.statistic_aggregation(
-            resource_id="VM_ID",
-            meter_name="cpu_util",
-            period="7300"
-        )
-        self.assertEqual(val, expected_result)
-
-    @mock.patch.object(CeilometerClient, "cmclient")
-    def test_get_last_sample(self, mock_keystone):
-        statistic = MagicMock()
-        expected_result = 100
-        statistic[-1]._info = {'counter_volume': expected_result}
-        mock_keystone.samples.list.return_value = statistic
-        val = self.cm.get_last_sample_value(
-            resource_id="id",
-            meter_name="compute.node.percent"
-        )
-        self.assertEqual(val, expected_result)
-
-    @mock.patch.object(CeilometerClient, "cmclient")
-    def test_get_last_sample_none(self, mock_keystone):
-        expected = []
-        mock_keystone.samples.list.return_value = expected
-        val = self.cm.get_last_sample_values(
-            resource_id="id",
-            meter_name="compute.node.percent"
-        )
-        self.assertEqual(val, expected)
-
-    @mock.patch.object(CeilometerClient, "cmclient")
-    def test_statistic_list(self, mock_keystone):
-        expected_value = []
-        mock_keystone.statistics.list.return_value = expected_value
-        val = self.cm.statistic_list(meter_name="cpu_util")
-        self.assertEqual(val, expected_value)

watcher/tests/common/test_ceilometer_helper.py

@@ -0,0 +1,98 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2015 b<>com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import mock
+from oslo_config import cfg
+
+from watcher.common import ceilometer_helper
+from watcher.common import clients
+from watcher.tests import base
+
+CONF = cfg.CONF
+
+
+@mock.patch.object(clients.OpenStackClients, 'ceilometer')
+class TestCeilometerHelper(base.BaseTestCase):
+
+    def test_build_query(self, mock_ceilometer):
+        mock_ceilometer.return_value = mock.MagicMock()
+        cm = ceilometer_helper.CeilometerHelper()
+        expected = [{'field': 'user_id', 'op': 'eq', 'value': u'user_id'},
+                    {'field': 'project_id', 'op': 'eq', 'value': u'tenant_id'},
+                    {'field': 'resource_id', 'op': 'eq',
+                     'value': u'resource_id'}]
+
+        query = cm.build_query(user_id="user_id",
+                               tenant_id="tenant_id",
+                               resource_id="resource_id",
+                               user_ids=["user_ids"],
+                               tenant_ids=["tenant_ids"],
+                               resource_ids=["resource_ids"])
+        self.assertEqual(query, expected)
+
+    def test_statistic_aggregation(self, mock_ceilometer):
+        cm = ceilometer_helper.CeilometerHelper()
+        ceilometer = mock.MagicMock()
+        statistic = mock.MagicMock()
+        expected_result = 100
+        statistic[-1]._info = {'aggregate': {'avg': expected_result}}
+        ceilometer.statistics.list.return_value = statistic
+        mock_ceilometer.return_value = ceilometer
+        cm = ceilometer_helper.CeilometerHelper()
+        val = cm.statistic_aggregation(
+            resource_id="VM_ID",
+            meter_name="cpu_util",
+            period="7300"
+        )
+        self.assertEqual(val, expected_result)
+
+    def test_get_last_sample(self, mock_ceilometer):
+        ceilometer = mock.MagicMock()
+        statistic = mock.MagicMock()
+        expected_result = 100
+        statistic[-1]._info = {'counter_volume': expected_result}
+        ceilometer.samples.list.return_value = statistic
+        mock_ceilometer.return_value = ceilometer
+        cm = ceilometer_helper.CeilometerHelper()
+        val = cm.get_last_sample_value(
+            resource_id="id",
+            meter_name="compute.node.percent"
+        )
+        self.assertEqual(val, expected_result)
+
+    def test_get_last_sample_none(self, mock_ceilometer):
+        ceilometer = mock.MagicMock()
+        expected = []
+        ceilometer.samples.list.return_value = expected
+        mock_ceilometer.return_value = ceilometer
+        cm = ceilometer_helper.CeilometerHelper()
+        val = cm.get_last_sample_values(
+            resource_id="id",
+            meter_name="compute.node.percent"
+        )
+        self.assertEqual(val, expected)
+
+    def test_statistic_list(self, mock_ceilometer):
+        ceilometer = mock.MagicMock()
+        expected_value = []
+        ceilometer.statistics.list.return_value = expected_value
+        mock_ceilometer.return_value = ceilometer
+        cm = ceilometer_helper.CeilometerHelper()
+        val = cm.statistic_list(meter_name="cpu_util")
+        self.assertEqual(val, expected_value)

watcher/tests/common/test_clients.py

@@ -0,0 +1,241 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from ceilometerclient import client as ceclient
+import ceilometerclient.v2.client as ceclient_v2
+from cinderclient import client as ciclient
+from cinderclient.v1 import client as ciclient_v1
+from glanceclient import client as glclient
+from keystoneauth1 import loading as ka_loading
+import mock
+from neutronclient.neutron import client as netclient
+from neutronclient.v2_0 import client as netclient_v2
+from novaclient import client as nvclient
+from oslo_config import cfg
+
+from watcher.common import clients
+from watcher.tests import base
+
+
+class TestClients(base.BaseTestCase):
+
+    def setUp(self):
+        super(TestClients, self).setUp()
+
+        cfg.CONF.import_opt('api_version', 'watcher.common.clients',
+                            group='nova_client')
+        cfg.CONF.import_opt('api_version', 'watcher.common.clients',
+                            group='glance_client')
+        cfg.CONF.import_opt('api_version', 'watcher.common.clients',
+                            group='cinder_client')
+        cfg.CONF.import_opt('api_version', 'watcher.common.clients',
+                            group='ceilometer_client')
+        cfg.CONF.import_opt('api_version', 'watcher.common.clients',
+                            group='neutron_client')
+
+    def test_get_keystone_session(self):
+        _AUTH_CONF_GROUP = 'watcher_clients_auth'
+        ka_loading.register_auth_conf_options(cfg.CONF, _AUTH_CONF_GROUP)
+        ka_loading.register_session_conf_options(cfg.CONF, _AUTH_CONF_GROUP)
+
+        cfg.CONF.set_override('auth_type', 'password',
+                              group=_AUTH_CONF_GROUP)
+
+        # If we don't clean up the _AUTH_CONF_GROUP conf options, then other
+        # tests that run after this one will fail, complaining about required
+        # options that _AUTH_CONF_GROUP wants.
+        def cleanup_conf_from_loading():
+            # oslo_config doesn't seem to allow unregistering groups through a
+            # single method, so we do this instead
+            cfg.CONF.reset()
+            del cfg.CONF._groups[_AUTH_CONF_GROUP]
+
+        self.addCleanup(cleanup_conf_from_loading)
+
+        osc = clients.OpenStackClients()
+
+        expected = {'username': 'foousername',
+                    'password': 'foopassword',
+                    'auth_url': 'http://server.ip:35357',
+                    'user_domain_id': 'foouserdomainid',
+                    'project_domain_id': 'fooprojdomainid'}
+
+        def reset_register_opts_mock(conf_obj, original_method):
+            conf_obj.register_opts = original_method
+
+        original_register_opts = cfg.CONF.register_opts
+        self.addCleanup(reset_register_opts_mock,
+                        cfg.CONF,
+                        original_register_opts)
+
+        # Because some of the conf options for auth plugins are not registered
+        # until right before they are loaded, and because the method that does
+        # the actual loading of the conf option values is an anonymous method
+        # (see _getter method of load_from_conf_options in
+        # keystoneauth1.loading.conf.py), we need to manually monkey patch
+        # the register opts method so that we can override the conf values to
+        # our custom values.
+        def mock_register_opts(*args, **kwargs):
+            ret = original_register_opts(*args, **kwargs)
+            if 'group' in kwargs and kwargs['group'] == _AUTH_CONF_GROUP:
+                for key, value in expected.items():
+                    cfg.CONF.set_override(key, value, group=_AUTH_CONF_GROUP)
+            return ret
+
+        cfg.CONF.register_opts = mock_register_opts
+
+        sess = osc.session
+        self.assertEqual(expected['auth_url'], sess.auth.auth_url)
+        self.assertEqual(expected['username'], sess.auth._username)
+        self.assertEqual(expected['password'], sess.auth._password)
+        self.assertEqual(expected['user_domain_id'], sess.auth._user_domain_id)
+        self.assertEqual(expected['project_domain_id'],
+                         sess.auth._project_domain_id)
+
+    @mock.patch.object(nvclient, 'Client')
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_nova(self, mock_session, mock_call):
+        osc = clients.OpenStackClients()
+        osc._nova = None
+        osc.nova()
+        mock_call.assert_called_once_with(cfg.CONF.nova_client.api_version,
+                                          session=mock_session)
+
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_nova_diff_vers(self, mock_session):
+        cfg.CONF.set_override('api_version', '2.3',
+                              group='nova_client')
+        osc = clients.OpenStackClients()
+        osc._nova = None
+        osc.nova()
+        self.assertEqual('2.3', osc.nova().api_version.get_string())
+
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_nova_cached(self, mock_session):
+        osc = clients.OpenStackClients()
+        osc._nova = None
+        nova = osc.nova()
+        nova_cached = osc.nova()
+        self.assertEqual(nova, nova_cached)
+
+    @mock.patch.object(glclient, 'Client')
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_glance(self, mock_session, mock_call):
+        osc = clients.OpenStackClients()
+        osc._glance = None
+        osc.glance()
+        mock_call.assert_called_once_with(cfg.CONF.glance_client.api_version,
+                                          session=mock_session)
+
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_glance_diff_vers(self, mock_session):
+        cfg.CONF.set_override('api_version', '1',
+                              group='glance_client')
+        osc = clients.OpenStackClients()
+        osc._glance = None
+        osc.glance()
+        self.assertEqual(1.0, osc.glance().version)
+
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_glance_cached(self, mock_session):
+        osc = clients.OpenStackClients()
+        osc._glance = None
+        glance = osc.glance()
+        glance_cached = osc.glance()
+        self.assertEqual(glance, glance_cached)
+
+    @mock.patch.object(ciclient, 'Client')
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_cinder(self, mock_session, mock_call):
+        osc = clients.OpenStackClients()
+        osc._cinder = None
+        osc.cinder()
+        mock_call.assert_called_once_with(cfg.CONF.cinder_client.api_version,
+                                          session=mock_session)
+
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_cinder_diff_vers(self, mock_session):
+        cfg.CONF.set_override('api_version', '1',
+                              group='cinder_client')
+        osc = clients.OpenStackClients()
+        osc._cinder = None
+        osc.cinder()
+        self.assertEqual(ciclient_v1.Client, type(osc.cinder()))
+
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_cinder_cached(self, mock_session):
+        osc = clients.OpenStackClients()
+        osc._cinder = None
+        cinder = osc.cinder()
+        cinder_cached = osc.cinder()
+        self.assertEqual(cinder, cinder_cached)
+
+    @mock.patch.object(ceclient, 'Client')
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_ceilometer(self, mock_session, mock_call):
+        osc = clients.OpenStackClients()
+        osc._ceilometer = None
+        osc.ceilometer()
+        mock_call.assert_called_once_with(
+            cfg.CONF.ceilometer_client.api_version,
+            session=mock_session)
+
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    @mock.patch.object(ceclient_v2.Client, '_get_alarm_client')
+    def test_clients_ceilometer_diff_vers(self, mock_get_alarm_client,
+                                          mock_session):
+        '''ceilometerclient currently only has one version (v2)'''
+        cfg.CONF.set_override('api_version', '2',
+                              group='ceilometer_client')
+        osc = clients.OpenStackClients()
+        osc._ceilometer = None
+        osc.ceilometer()
+        self.assertEqual(ceclient_v2.Client,
+                         type(osc.ceilometer()))
+
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    @mock.patch.object(ceclient_v2.Client, '_get_alarm_client')
+    def test_clients_ceilometer_cached(self, mock_get_alarm_client,
+                                       mock_session):
+        osc = clients.OpenStackClients()
+        osc._ceilometer = None
+        ceilometer = osc.ceilometer()
+        ceilometer_cached = osc.ceilometer()
+        self.assertEqual(ceilometer, ceilometer_cached)
+
+    @mock.patch.object(netclient, 'Client')
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_neutron(self, mock_session, mock_call):
+        osc = clients.OpenStackClients()
+        osc._neutron = None
+        osc.neutron()
+        mock_call.assert_called_once_with(cfg.CONF.neutron_client.api_version,
+                                          session=mock_session)
+
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_neutron_diff_vers(self, mock_session):
+        '''neutronclient currently only has one version (v2)'''
+        cfg.CONF.set_override('api_version', '2',
+                              group='neutron_client')
+        osc = clients.OpenStackClients()
+        osc._neutron = None
+        osc.neutron()
+        self.assertEqual(netclient_v2.Client,
+                         type(osc.neutron()))
+
+    @mock.patch.object(clients.OpenStackClients, 'session')
+    def test_clients_neutron_cached(self, mock_session):
+        osc = clients.OpenStackClients()
+        osc._neutron = None
+        neutron = osc.neutron()
+        neutron_cached = osc.neutron()
+        self.assertEqual(neutron, neutron_cached)

watcher/tests/common/test_keystone.py

@@ -1,68 +0,0 @@
-# -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import unicode_literals
-from keystoneclient.auth.identity import Password
-from keystoneclient.session import Session
-from mock import mock
-from oslo_config import cfg
-from watcher.common.keystone import KeystoneClient
-from watcher.tests.base import BaseTestCase
-
-CONF = cfg.CONF
-
-
-class TestKeystone(BaseTestCase):
-    def setUp(self):
-        super(TestKeystone, self).setUp()
-        self.ckeystone = KeystoneClient()
-
-    @mock.patch('keystoneclient.v2_0.client.Client', autospec=True)
-    def test_get_endpoint_v2(self, keystone):
-        expected_endpoint = "http://ip:port/v2"
-        cfg.CONF.set_override(
-            'auth_uri', expected_endpoint, group="keystone_authtoken",
-            enforce_type=True
-        )
-        ks = mock.Mock()
-        ks.service_catalog.url_for.return_value = expected_endpoint
-        keystone.return_value = ks
-        ep = self.ckeystone.get_endpoint(service_type='metering',
-                                         endpoint_type='publicURL',
-                                         region_name='RegionOne')
-
-        self.assertEqual(ep, expected_endpoint)
-
-    @mock.patch('watcher.common.keystone.KeystoneClient._is_apiv3')
-    def test_get_session(self, mock_apiv3):
-        mock_apiv3.return_value = True
-        k = KeystoneClient()
-        session = k.get_session()
-        self.assertIsInstance(session.auth, Password)
-        self.assertIsInstance(session, Session)
-
-    @mock.patch('watcher.common.keystone.KeystoneClient._is_apiv3')
-    def test_get_credentials(self, mock_apiv3):
-        mock_apiv3.return_value = True
-        expected_creds = {'auth_url': None,
-                          'password': None,
-                          'project_domain_name': 'default',
-                          'project_name': 'admin',
-                          'user_domain_name': 'default',
-                          'username': None}
-        creds = self.ckeystone.get_credentials()
-        self.assertEqual(creds, expected_creds)

watcher/tests/common/test_nova_client.py

@@ -1,153 +0,0 @@
-# -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
-#
-# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-
-import glanceclient.v2.client as glclient
-import mock
-import novaclient.client as nvclient
-
-from watcher.common import keystone
-from watcher.common.nova import NovaClient
-from watcher.common import utils
-from watcher.tests import base
-
-
-class TestNovaClient(base.TestCase):
-
-    def setUp(self):
-        super(TestNovaClient, self).setUp()
-        self.instance_uuid = "fb5311b7-37f3-457e-9cde-6494a3c59bfe"
-        self.source_hypervisor = "ldev-indeedsrv005"
-        self.destination_hypervisor = "ldev-indeedsrv006"
-
-        self.creds = mock.MagicMock()
-        self.session = mock.MagicMock()
-
-    @mock.patch.object(keystone, 'KeystoneClient', mock.Mock())
-    @mock.patch.object(nvclient, "Client", mock.Mock())
-    def test_stop_instance(self):
-        nova_client = NovaClient(creds=self.creds, session=self.session)
-        instance_id = utils.generate_uuid()
-        server = mock.MagicMock()
-        server.id = instance_id
-        setattr(server, 'OS-EXT-STS:vm_state', 'stopped')
-        nova_client.nova.servers = mock.MagicMock()
-        nova_client.nova.servers.find.return_value = server
-        nova_client.nova.servers.list.return_value = [server]
-
-        result = nova_client.stop_instance(instance_id)
-        self.assertEqual(result, True)
-
-    @mock.patch.object(keystone, 'KeystoneClient', mock.Mock())
-    @mock.patch.object(nvclient, "Client", mock.Mock())
-    def test_set_host_offline(self):
-        nova_client = NovaClient(creds=self.creds, session=self.session)
-        host = mock.MagicMock()
-        nova_client.nova.hosts = mock.MagicMock()
-        nova_client.nova.hosts.get.return_value = host
-        result = nova_client.set_host_offline("rennes")
-        self.assertEqual(result, True)
-
-    @mock.patch.object(time, 'sleep', mock.Mock())
-    @mock.patch.object(keystone, 'KeystoneClient', mock.Mock())
-    @mock.patch.object(nvclient, "Client", mock.Mock())
-    def test_live_migrate_instance(self):
-        nova_client = NovaClient(creds=self.creds, session=self.session)
-        server = mock.MagicMock()
-        server.id = self.instance_uuid
-        nova_client.nova.servers = mock.MagicMock()
-        nova_client.nova.servers.list.return_value = [server]
-        instance = nova_client.live_migrate_instance(
-            self.instance_uuid, self.destination_hypervisor
-        )
-        self.assertIsNotNone(instance)
-
-    @mock.patch.object(keystone, 'KeystoneClient', mock.Mock())
-    @mock.patch.object(nvclient, "Client", mock.Mock())
-    def test_watcher_non_live_migrate_instance_not_found(self):
-        nova_client = NovaClient(creds=self.creds, session=self.session)
-        nova_client.nova.servers.list.return_value = []
-        nova_client.nova.servers.find.return_value = None
-
-        is_success = nova_client.watcher_non_live_migrate_instance(
-            self.instance_uuid,
-            self.destination_hypervisor)
-
-        self.assertEqual(is_success, False)
-
-    @mock.patch.object(time, 'sleep', mock.Mock())
-    @mock.patch.object(keystone, 'KeystoneClient', mock.Mock())
-    @mock.patch.object(nvclient, "Client", mock.Mock())
-    def test_watcher_non_live_migrate_instance_volume(self):
-        nova_client = NovaClient(creds=self.creds, session=self.session)
-        instance = mock.MagicMock(id=self.instance_uuid)
-        setattr(instance, 'OS-EXT-SRV-ATTR:host', self.source_hypervisor)
-        nova_client.nova.servers.list.return_value = [instance]
-        nova_client.nova.servers.find.return_value = instance
-        instance = nova_client.watcher_non_live_migrate_instance(
-            self.instance_uuid,
-            self.destination_hypervisor)
-        self.assertIsNotNone(instance)
-
-    @mock.patch.object(time, 'sleep', mock.Mock())
-    @mock.patch.object(keystone, 'KeystoneClient', mock.Mock())
-    @mock.patch.object(nvclient, "Client", mock.Mock())
-    def test_watcher_non_live_migrate_keep_image(self):
-        nova_client = NovaClient(creds=self.creds, session=self.session)
-        instance = mock.MagicMock(id=self.instance_uuid)
-        setattr(instance, 'OS-EXT-SRV-ATTR:host', self.source_hypervisor)
-        addresses = mock.MagicMock()
-        type = mock.MagicMock()
-        networks = []
-        networks.append(("lan", type))
-        addresses.items.return_value = networks
-        attached_volumes = mock.MagicMock()
-        setattr(instance, 'addresses', addresses)
-        setattr(instance, "os-extended-volumes:volumes_attached",
-                attached_volumes)
-        nova_client.nova.servers.list.return_value = [instance]
-        nova_client.nova.servers.find.return_value = instance
-        instance = nova_client.watcher_non_live_migrate_instance(
-            self.instance_uuid,
-            self.destination_hypervisor, keep_original_image_name=False)
-        self.assertIsNotNone(instance)
-
-    @mock.patch.object(time, 'sleep', mock.Mock())
-    @mock.patch.object(keystone, 'KeystoneClient', mock.Mock())
-    @mock.patch.object(nvclient, "Client", mock.Mock())
-    @mock.patch.object(glclient, "Client")
-    def test_create_image_from_instance(self, m_glance_cls):
-        nova_client = NovaClient(creds=self.creds, session=self.session)
-        instance = mock.MagicMock()
-        image = mock.MagicMock()
-        setattr(instance, 'OS-EXT-SRV-ATTR:host', self.source_hypervisor)
-        nova_client.nova.servers.list.return_value = [instance]
-        nova_client.nova.servers.find.return_value = instance
-        image_uuid = 'fake-image-uuid'
-        nova_client.nova.servers.create_image.return_value = image
-
-        m_glance = mock.MagicMock()
-        m_glance_cls.return_value = m_glance
-
-        m_glance.images = {image_uuid: image}
-        instance = nova_client.create_image_from_instance(
-            self.instance_uuid, "Cirros"
-        )
-        self.assertIsNone(instance)

watcher/tests/common/test_nova_helper.py

@@ -0,0 +1,144 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2015 b<>com
+#
+# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+
+import mock
+
+from watcher.common import clients
+from watcher.common import nova_helper
+from watcher.common import utils
+from watcher.tests import base
+
+
+@mock.patch.object(clients.OpenStackClients, 'nova')
+@mock.patch.object(clients.OpenStackClients, 'neutron')
+@mock.patch.object(clients.OpenStackClients, 'cinder')
+@mock.patch.object(clients.OpenStackClients, 'glance')
+class TestNovaHelper(base.TestCase):
+
+    def setUp(self):
+        super(TestNovaHelper, self).setUp()
+        self.instance_uuid = "fb5311b7-37f3-457e-9cde-6494a3c59bfe"
+        self.source_hypervisor = "ldev-indeedsrv005"
+        self.destination_hypervisor = "ldev-indeedsrv006"
+
+    def test_stop_instance(self, mock_glance, mock_cinder, mock_neutron,
+                           mock_nova):
+        nova_util = nova_helper.NovaHelper()
+        instance_id = utils.generate_uuid()
+        server = mock.MagicMock()
+        server.id = instance_id
+        setattr(server, 'OS-EXT-STS:vm_state', 'stopped')
+        nova_util.nova.servers = mock.MagicMock()
+        nova_util.nova.servers.find.return_value = server
+        nova_util.nova.servers.list.return_value = [server]
+
+        result = nova_util.stop_instance(instance_id)
+        self.assertEqual(result, True)
+
+    def test_set_host_offline(self, mock_glance, mock_cinder, mock_neutron,
+                              mock_nova):
+        nova_util = nova_helper.NovaHelper()
+        host = mock.MagicMock()
+        nova_util.nova.hosts = mock.MagicMock()
+        nova_util.nova.hosts.get.return_value = host
+        result = nova_util.set_host_offline("rennes")
+        self.assertEqual(result, True)
+
+    @mock.patch.object(time, 'sleep', mock.Mock())
+    def test_live_migrate_instance(self, mock_glance, mock_cinder,
+                                   mock_neutron, mock_nova):
+        nova_util = nova_helper.NovaHelper()
+        server = mock.MagicMock()
+        server.id = self.instance_uuid
+        nova_util.nova.servers = mock.MagicMock()
+        nova_util.nova.servers.list.return_value = [server]
+        instance = nova_util.live_migrate_instance(
+            self.instance_uuid, self.destination_hypervisor
+        )
+        self.assertIsNotNone(instance)
+
+    def test_watcher_non_live_migrate_instance_not_found(
+            self, mock_glance, mock_cinder, mock_neutron, mock_nova):
+        nova_util = nova_helper.NovaHelper()
+        nova_util.nova.servers.list.return_value = []
+        nova_util.nova.servers.find.return_value = None
+
+        is_success = nova_util.watcher_non_live_migrate_instance(
+            self.instance_uuid,
+            self.destination_hypervisor)
+
+        self.assertEqual(is_success, False)
+
+    @mock.patch.object(time, 'sleep', mock.Mock())
+    def test_watcher_non_live_migrate_instance_volume(
+            self, mock_glance, mock_cinder, mock_neutron, mock_nova):
+        nova_util = nova_helper.NovaHelper()
+        instance = mock.MagicMock(id=self.instance_uuid)
+        setattr(instance, 'OS-EXT-SRV-ATTR:host', self.source_hypervisor)
+        nova_util.nova.servers.list.return_value = [instance]
+        nova_util.nova.servers.find.return_value = instance
+        instance = nova_util.watcher_non_live_migrate_instance(
+            self.instance_uuid,
+            self.destination_hypervisor)
+        self.assertIsNotNone(instance)
+
+    @mock.patch.object(time, 'sleep', mock.Mock())
+    def test_watcher_non_live_migrate_keep_image(
+            self, mock_glance, mock_cinder, mock_neutron, mock_nova):
+        nova_util = nova_helper.NovaHelper()
+        instance = mock.MagicMock(id=self.instance_uuid)
+        setattr(instance, 'OS-EXT-SRV-ATTR:host', self.source_hypervisor)
+        addresses = mock.MagicMock()
+        type = mock.MagicMock()
+        networks = []
+        networks.append(("lan", type))
+        addresses.items.return_value = networks
+        attached_volumes = mock.MagicMock()
+        setattr(instance, 'addresses', addresses)
+        setattr(instance, "os-extended-volumes:volumes_attached",
+                attached_volumes)
+        nova_util.nova.servers.list.return_value = [instance]
+        nova_util.nova.servers.find.return_value = instance
+        instance = nova_util.watcher_non_live_migrate_instance(
+            self.instance_uuid,
+            self.destination_hypervisor, keep_original_image_name=False)
+        self.assertIsNotNone(instance)
+
+    @mock.patch.object(time, 'sleep', mock.Mock())
+    def test_create_image_from_instance(self, mock_glance, mock_cinder,
+                                        mock_neutron, mock_nova):
+        nova_util = nova_helper.NovaHelper()
+        instance = mock.MagicMock()
+        image = mock.MagicMock()
+        setattr(instance, 'OS-EXT-SRV-ATTR:host', self.source_hypervisor)
+        nova_util.nova.servers.list.return_value = [instance]
+        nova_util.nova.servers.find.return_value = instance
+        image_uuid = 'fake-image-uuid'
+        nova_util.nova.servers.create_image.return_value = image
+
+        glance_client = mock.MagicMock()
+        mock_glance.return_value = glance_client
+
+        glance_client.images = {image_uuid: image}
+        instance = nova_util.create_image_from_instance(
+            self.instance_uuid, "Cirros"
+        )
+        self.assertIsNone(instance)

watcher/tests/decision_engine/strategy/selector/test_strategy_selector.py

@@ -36,8 +36,8 @@ class TestStrategySelector(TestCase):
                               enforce_type=True)
         expected_goal = 'DUMMY'
         expected_strategy = CONF.watcher_goals.goals[expected_goal]
-        self.strategy_selector.define_from_goal(expected_goal)
-        mock_call.assert_called_once_with(expected_strategy)
+        self.strategy_selector.define_from_goal(expected_goal, osc=None)
+        mock_call.assert_called_once_with(expected_strategy, osc=None)
 
     @patch.object(DefaultStrategyLoader, 'load')
     def test_define_from_goal_with_incorrect_mapping(self, mock_call):