From 9a6811ae6bd07580d2e485b6312821919451dfa4 Mon Sep 17 00:00:00 2001 From: Taylor Peoples Date: Wed, 20 Jan 2016 08:15:47 +0100 Subject: [PATCH] Create OpenStackClients convenience class The OpenStackClients class provides a convenient way to create and cache client instances. The idea behind this code comes from Magnum [0]. The OpenStackClients class will act as the manager of other project's clients, providing an easy way to fetch instances of said clients. This will allow the clients to be cached. An instance of OpenStackClients is created for every call that comes into the decision engine and the applier, using the request context to pass needed (domain id) parameters to get a Keystone session. This instance should be shared as much as possible to avoid additional unneccessary connections to the other services. This class will also allow for the version of each client to be configurable via the watcher.conf file. The method by which a Keystone session is also changed to use the keystoneauth1.loading library. In order to avoid DuplicateOptErrors with the keystone_authtoken group used for the keystonemiddleware in the API code, a new conf group named "watcher_clients_auth" is created. A typical configuration using a password authentication scheme will look like: [watcher_clients_auth] auth_type = password auth_url = http://: username = password = project_domain_id = default user_domain_id = default [0]: https://github.com/openstack/magnum/blob/master/magnum/common/clients.py DocImpact Change-Id: Iab9d0b304099686da2e9e2b19e8b1de4332ff378 Implements: blueprint external-api-versioning Closes-Bug: #1530790 Closes-Bug: #1539670 Closes-Bug: #1522774 --- devstack/lib/watcher | 8 +- doc/source/deploy/configuration.rst | 52 +- etc/watcher/watcher.conf.sample | 919 ++++++++++-------- requirements.txt | 1 + watcher/applier/action_plan/default.py | 2 +- watcher/applier/actions/base.py | 12 +- .../actions/change_nova_service_state.py | 11 +- watcher/applier/actions/factory.py | 5 +- watcher/applier/actions/migration.py | 13 +- watcher/applier/default.py | 9 +- watcher/applier/workflow_engine/base.py | 20 +- watcher/applier/workflow_engine/default.py | 4 +- .../{ceilometer.py => ceilometer_helper.py} | 41 +- watcher/common/clients.py | 158 +++ watcher/common/exception.py | 25 + watcher/common/keystone.py | 131 --- watcher/common/loader/default.py | 4 +- watcher/common/{nova.py => nova_helper.py} | 39 +- .../strategy/context/default.py | 9 +- .../strategy/selection/default.py | 5 +- .../strategy/strategies/base.py | 13 +- .../strategies/basic_consolidation.py | 8 +- .../strategy/strategies/dummy_strategy.py | 5 +- .../strategies/outlet_temp_control.py | 8 +- .../cluster_history/ceilometer.py | 8 +- .../cluster_model_collector/manager.py | 12 +- watcher/opts.py | 17 +- .../test_default_workflow_engine.py | 6 +- watcher/tests/common/test_ceilometer.py | 99 -- .../tests/common/test_ceilometer_helper.py | 98 ++ watcher/tests/common/test_clients.py | 241 +++++ watcher/tests/common/test_keystone.py | 68 -- watcher/tests/common/test_nova_client.py | 153 --- watcher/tests/common/test_nova_helper.py | 144 +++ .../selector/test_strategy_selector.py | 4 +- 35 files changed, 1373 insertions(+), 979 deletions(-) rename watcher/common/{ceilometer.py => ceilometer_helper.py} (80%) create mode 100644 watcher/common/clients.py delete mode 100644 watcher/common/keystone.py rename watcher/common/{nova.py => nova_helper.py} (95%) delete mode 100644 watcher/tests/common/test_ceilometer.py create mode 100644 watcher/tests/common/test_ceilometer_helper.py create mode 100644 watcher/tests/common/test_clients.py delete mode 100644 watcher/tests/common/test_keystone.py delete mode 100644 watcher/tests/common/test_nova_client.py create mode 100644 watcher/tests/common/test_nova_helper.py diff --git a/devstack/lib/watcher b/devstack/lib/watcher index 3aa8bad78..cd07aaef7 100644 --- a/devstack/lib/watcher +++ b/devstack/lib/watcher @@ -128,14 +128,8 @@ function create_watcher_conf { iniset $WATCHER_CONF oslo_messaging_rabbit rabbit_password $RABBIT_PASSWORD iniset $WATCHER_CONF oslo_messaging_rabbit rabbit_host $RABBIT_HOST - iniset $WATCHER_CONF keystone_authtoken admin_user watcher - iniset $WATCHER_CONF keystone_authtoken admin_password $SERVICE_PASSWORD - iniset $WATCHER_CONF keystone_authtoken admin_tenant_name $SERVICE_TENANT_NAME - configure_auth_token_middleware $WATCHER_CONF watcher $WATCHER_AUTH_CACHE_DIR - - iniset $WATCHER_CONF keystone_authtoken auth_uri $KEYSTONE_SERVICE_URI/v3 - iniset $WATCHER_CONF keystone_authtoken auth_version v3 + configure_auth_token_middleware $WATCHER_CONF watcher $WATCHER_AUTH_CACHE_DIR "watcher_clients_auth" if is_fedora || is_suse; then # watcher defaults to /usr/local/bin, but fedora and suse pip like to diff --git a/doc/source/deploy/configuration.rst b/doc/source/deploy/configuration.rst index 533249153..cb288b9ba 100644 --- a/doc/source/deploy/configuration.rst +++ b/doc/source/deploy/configuration.rst @@ -166,11 +166,17 @@ The configuration file is organized into the following sections: * ``[api]`` - API server configuration * ``[database]`` - SQL driver configuration * ``[keystone_authtoken]`` - Keystone Authentication plugin configuration +* ``[watcher_clients_auth]`` - Keystone auth configuration for clients * ``[watcher_applier]`` - Watcher Applier module configuration * ``[watcher_decision_engine]`` - Watcher Decision Engine module configuration * ``[watcher_goals]`` - Goals mapping configuration * ``[watcher_strategies]`` - Strategy configuration * ``[oslo_messaging_rabbit]`` - Oslo Messaging RabbitMQ driver configuration +* ``[ceilometer_client]`` - Ceilometer client configuration +* ``[cinder_client]`` - Cinder client configuration +* ``[glance_client]`` - Glance client configuration +* ``[nova_client]`` - Nova client configuration +* ``[neutron_client]`` - Neutron client configuration The Watcher configuration file is expected to be named ``watcher.conf``. When starting Watcher, you can specify a different @@ -246,7 +252,11 @@ so that the watcher service is configured for your needs. # Complete public Identity API endpoint (string value) #auth_uri= - auth_uri=http://IDENTITY_IP:5000/v3 + auth_uri=http://IDENTITY_IP:5000/ + + # API version of the admin Identity API endpoint. (string value) + #auth_version= + auth_version=v3 # Complete admin Identity API endpoint. This should specify the # unversioned root endpoint e.g. https://localhost:35357/ (string @@ -271,6 +281,46 @@ so that the watcher service is configured for your needs. # value) #signing_dir= +#. Configure the credentials to use to authenticate with the Identity Service + for the different project clients:: + + [watcher_clients_auth] + + # Authentication type to load (unknown value) + # Deprecated group/name - [DEFAULT]/auth_plugin + #auth_type = + auth_type = password + + # Authentication URL (unknown value) + #auth_url = + auth_url = http://IDENTITY_IP:35357 + + # Username (unknown value) + # Deprecated group/name - [DEFAULT]/username + #username = + username=watcher + + # User's password (unknown value) + #password = + password = WATCHER_PASSWORD + + # Domain ID containing project (unknown value) + #project_domain_id = + project_domain_id = default + + # User's domain id (unknown value) + #user_domain_id = + user_domain_id = default + +#. Configure the clients to use a specific version if desired. For example, to + configure Watcher to use a Nova client with version 2.1, use:: + + [nova_client] + + # Version of Nova API to use in novaclient. (string value) + #api_version = 2 + api_version = 2.1 + #. Create the Watcher Service database tables:: $ watcher-db-manage --config-file /etc/watcher/watcher.conf create_schema diff --git a/etc/watcher/watcher.conf.sample b/etc/watcher/watcher.conf.sample index 795574a34..85aecfd24 100644 --- a/etc/watcher/watcher.conf.sample +++ b/etc/watcher/watcher.conf.sample @@ -4,6 +4,42 @@ # From oslo.log # +# Enables or disables fatal status of deprecations. (boolean value) +#fatal_deprecations = false + +# DEPRECATED. A logging.Formatter log message format string which may +# use any of the available logging.LogRecord attributes. This option +# is deprecated. Please use logging_context_format_string and +# logging_default_format_string instead. This option is ignored if +# log_config_append is set. (string value) +#log_format = + +# Defines the format string for %%(asctime)s in log records. Default: +# %(default)s . This option is ignored if log_config_append is set. +# (string value) +#log_date_format = %Y-%m-%d %H:%M:%S + +# Enables or disables publication of error events. (boolean value) +#publish_errors = false + +# (Optional) Name of log file to send logging output to. If no default +# is set, logging will go to stderr as defined by use_stderr. This +# option is ignored if log_config_append is set. (string value) +# Deprecated group/name - [DEFAULT]/logfile +#log_file = + +# (Optional) The base directory used for relative log_file paths. +# This option is ignored if log_config_append is set. (string value) +# Deprecated group/name - [DEFAULT]/logdir +#log_dir = + +# Uses logging handler designed to watch file system. When log file is +# moved or removed this handler will open a new log file with +# specified path instantaneously. It makes sense only if log_file +# option is specified and Linux platform is used. This option is +# ignored if log_config_append is set. (boolean value) +#watch_log_file = false + # Use syslog for logging. Existing syslog format is DEPRECATED and # will be changed later to honor RFC5424. This option is ignored if # log_config_append is set. (boolean value) @@ -18,11 +54,6 @@ # Its value may be silently ignored in the future. #use_syslog_rfc_format = true -# (Optional) The base directory used for relative log_file paths. -# This option is ignored if log_config_append is set. (string value) -# Deprecated group/name - [DEFAULT]/logdir -#log_dir = - # Syslog facility to receive log lines. This option is ignored if # log_config_append is set. (string value) #syslog_log_facility = LOG_USER @@ -42,23 +73,31 @@ # message is DEBUG. (string value) #logging_debug_format_suffix = %(funcName)s %(pathname)s:%(lineno)d -# If set to true, the logging level will be set to DEBUG instead of -# the default INFO level. (boolean value) -#debug = false - # Prefix each line of exception output with this format. (string # value) #logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d ERROR %(name)s %(instance)s +# Defines the format string for %(user_identity)s that is used in +# logging_context_format_string. (string value) +#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s + # If set to false, the logging level will be set to WARNING instead of # the default INFO level. (boolean value) # This option is deprecated for removal. # Its value may be silently ignored in the future. #verbose = true -# Defines the format string for %(user_identity)s that is used in -# logging_context_format_string. (string value) -#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s +# List of package logging levels in logger=LEVEL pairs. This option is +# ignored if log_config_append is set. (list value) +#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN + +# If set to true, the logging level will be set to DEBUG instead of +# the default INFO level. (boolean value) +#debug = false + +# The format for an instance that is passed with the log message. +# (string value) +#instance_format = "[instance: %(uuid)s] " # The name of a logging configuration file. This file is appended to # any existing logging configuration files. For details about logging @@ -70,68 +109,65 @@ # Deprecated group/name - [DEFAULT]/log_config #log_config_append = -# List of package logging levels in logger=LEVEL pairs. This option is -# ignored if log_config_append is set. (list value) -#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN - -# DEPRECATED. A logging.Formatter log message format string which may -# use any of the available logging.LogRecord attributes. This option -# is deprecated. Please use logging_context_format_string and -# logging_default_format_string instead. This option is ignored if -# log_config_append is set. (string value) -#log_format = - -# Enables or disables publication of error events. (boolean value) -#publish_errors = false - -# Defines the format string for %%(asctime)s in log records. Default: -# %(default)s . This option is ignored if log_config_append is set. -# (string value) -#log_date_format = %Y-%m-%d %H:%M:%S - -# The format for an instance that is passed with the log message. -# (string value) -#instance_format = "[instance: %(uuid)s] " - -# (Optional) Name of log file to send logging output to. If no default -# is set, logging will go to stderr as defined by use_stderr. This -# option is ignored if log_config_append is set. (string value) -# Deprecated group/name - [DEFAULT]/logfile -#log_file = - # The format for an instance UUID that is passed with the log message. # (string value) #instance_uuid_format = "[instance: %(uuid)s] " -# Enables or disables fatal status of deprecations. (boolean value) -#fatal_deprecations = false - -# Uses logging handler designed to watch file system. When log file is -# moved or removed this handler will open a new log file with -# specified path instantaneously. It makes sense only if log_file -# option is specified and Linux platform is used. This option is -# ignored if log_config_append is set. (boolean value) -#watch_log_file = false - # # From oslo.messaging # -# Directory for holding IPC sockets. (string value) -#rpc_zmq_ipc_dir = /var/run/openstack +# The default number of seconds that poll should wait. Poll raises +# timeout exception when timeout expired. (integer value) +#rpc_poll_timeout = 1 -# Number of retries to find free port number before fail with -# ZMQBindError. (integer value) -#rpc_zmq_bind_port_retries = 100 +# Name of this node. Must be a valid hostname, FQDN, or IP address. +# Must match "host" option, if running Nova. (string value) +#rpc_zmq_host = localhost + +# Configures zmq-messaging to use proxy with non PUB/SUB patterns. +# (boolean value) +#direct_over_proxy = true # AMQP topic used for OpenStack notifications. (list value) # Deprecated group/name - [rpc_notifier2]/topics # Deprecated group/name - [DEFAULT]/notification_topics #topics = notifications -# Name of this node. Must be a valid hostname, FQDN, or IP address. -# Must match "host" option, if running Nova. (string value) -#rpc_zmq_host = localhost +# Use PUB/SUB pattern for fanout methods. PUB/SUB always uses proxy. +# (boolean value) +#use_pub_sub = true + +# Minimal port number for random ports range. (port value) +# Minimum value: 0 +# Maximum value: 65535 +#rpc_zmq_min_port = 49152 + +# Seconds to wait for a response from a call. (integer value) +#rpc_response_timeout = 60 + +# Maximal port number for random ports range. (integer value) +# Minimum value: 1 +# Maximum value: 65536 +#rpc_zmq_max_port = 65536 + +# A URL representing the messaging driver to use and its full +# configuration. If not set, we fall back to the rpc_backend option +# and driver specific configuration. (string value) +#transport_url = + +# Use this port to connect to redis host. (port value) +# Minimum value: 0 +# Maximum value: 65535 +#port = 6379 + +# Number of retries to find free port number before fail with +# ZMQBindError. (integer value) +#rpc_zmq_bind_port_retries = 100 + +# Size of RPC connection pool. (integer value) +# Deprecated group/name - [DEFAULT]/rpc_conn_pool_size +#rpc_conn_pool_size = 30 # The messaging driver to use, defaults to rabbit. Other drivers # include amqp and zmq. (string value) @@ -140,50 +176,48 @@ # Host to locate redis. (string value) #host = 127.0.0.1 -# Seconds to wait before a cast expires (TTL). Only supported by -# impl_zmq. (integer value) -#rpc_cast_timeout = 30 +# The default exchange under which topics are scoped. May be +# overridden by an exchange name specified in the transport_url +# option. (string value) +#control_exchange = openstack -# Seconds to wait for a response from a call. (integer value) -#rpc_response_timeout = 60 +# MatchMaker driver. (string value) +#rpc_zmq_matchmaker = redis -# Use this port to connect to redis host. (port value) -# Minimum value: 1 -# Maximum value: 65535 -#port = 6379 - -# The default number of seconds that poll should wait. Poll raises -# timeout exception when timeout expired. (integer value) -#rpc_poll_timeout = 1 - -# A URL representing the messaging driver to use and its full -# configuration. If not set, we fall back to the rpc_backend option -# and driver specific configuration. (string value) -#transport_url = +# Type of concurrency used. Either "native" or "eventlet" (string +# value) +#rpc_zmq_concurrency = eventlet # Password for Redis server (optional). (string value) #password = -# Configures zmq-messaging to use proxy with non PUB/SUB patterns. -# (boolean value) -#direct_over_proxy = true +# Number of ZeroMQ contexts, defaults to 1. (integer value) +#rpc_zmq_contexts = 1 + +# Maximum number of ingress messages to locally buffer per topic. +# Default is unlimited. (integer value) +#rpc_zmq_topic_backlog = + +# Size of executor thread pool. (integer value) +# Deprecated group/name - [DEFAULT]/rpc_thread_pool_size +#executor_thread_pool_size = 64 + +# Directory for holding IPC sockets. (string value) +#rpc_zmq_ipc_dir = /var/run/openstack # The Drivers(s) to handle sending notifications. Possible values are # messaging, messagingv2, routing, log, test, noop (multi valued) # Deprecated group/name - [DEFAULT]/notification_driver #driver = -# Size of executor thread pool. (integer value) -# Deprecated group/name - [DEFAULT]/rpc_thread_pool_size -#executor_thread_pool_size = 64 +# ZeroMQ bind address. Should be a wildcard (*), an ethernet +# interface, or IP. The "host" option should point or resolve to this +# address. (string value) +#rpc_zmq_bind_address = * -# Size of RPC connection pool. (integer value) -# Deprecated group/name - [DEFAULT]/rpc_conn_pool_size -#rpc_conn_pool_size = 30 - -# Use PUB/SUB pattern for fanout methods. PUB/SUB always uses proxy. -# (boolean value) -#use_pub_sub = true +# Seconds to wait before a cast expires (TTL). Only supported by +# impl_zmq. (integer value) +#rpc_cast_timeout = 30 # A URL representing the messaging driver to use for notifications. If # not set, we fall back to the same configuration used for RPC. @@ -191,40 +225,6 @@ # Deprecated group/name - [DEFAULT]/notification_transport_url #transport_url = -# ZeroMQ bind address. Should be a wildcard (*), an ethernet -# interface, or IP. The "host" option should point or resolve to this -# address. (string value) -#rpc_zmq_bind_address = * - -# MatchMaker driver. (string value) -#rpc_zmq_matchmaker = redis - -# Minimal port number for random ports range. (port value) -# Minimum value: 1 -# Maximum value: 65535 -#rpc_zmq_min_port = 49152 - -# Type of concurrency used. Either "native" or "eventlet" (string -# value) -#rpc_zmq_concurrency = eventlet - -# Number of ZeroMQ contexts, defaults to 1. (integer value) -#rpc_zmq_contexts = 1 - -# Maximal port number for random ports range. (integer value) -# Minimum value: 1 -# Maximum value: 65536 -#rpc_zmq_max_port = 65536 - -# The default exchange under which topics are scoped. May be -# overridden by an exchange name specified in the transport_url -# option. (string value) -#control_exchange = openstack - -# Maximum number of ingress messages to locally buffer per topic. -# Default is unlimited. (integer value) -#rpc_zmq_topic_backlog = - [api] @@ -232,33 +232,100 @@ # From watcher # -# The port for the watcher API server (integer value) -#port = 9322 - # The listen IP for the watcher API server (string value) #host = 0.0.0.0 +# The port for the watcher API server (integer value) +#port = 9322 + # The maximum number of items returned in a single response from a # collection resource. (integer value) #max_limit = 1000 +[ceilometer_client] + +# +# From watcher +# + +# Version of Ceilometer API to use in ceilometerclient. (string value) +#api_version = 2 + + +[cinder_client] + +# +# From watcher +# + +# Version of Cinder API to use in cinderclient. (string value) +#api_version = 2 + + [database] # # From oslo.db # -# The SQLAlchemy connection string to use to connect to the database. -# (string value) -# Deprecated group/name - [DEFAULT]/sql_connection -# Deprecated group/name - [DATABASE]/sql_connection -# Deprecated group/name - [sql]/connection -#connection = +# If set, use this value for max_overflow with SQLAlchemy. (integer +# value) +# Deprecated group/name - [DEFAULT]/sql_max_overflow +# Deprecated group/name - [DATABASE]/sqlalchemy_max_overflow +#max_overflow = -# Add Python stack traces to SQL as comment strings. (boolean value) -# Deprecated group/name - [DEFAULT]/sql_connection_trace -#connection_trace = false +# Maximum number of SQL connections to keep open in a pool. (integer +# value) +# Deprecated group/name - [DEFAULT]/sql_max_pool_size +# Deprecated group/name - [DATABASE]/sql_max_pool_size +#max_pool_size = + +# Interval between retries of opening a SQL connection. (integer +# value) +# Deprecated group/name - [DEFAULT]/sql_retry_interval +# Deprecated group/name - [DATABASE]/reconnect_interval +#retry_interval = 10 + +# Enable the experimental use of database reconnect on connection +# lost. (boolean value) +#use_db_reconnect = false + +# Minimum number of SQL connections to keep open in a pool. (integer +# value) +# Deprecated group/name - [DEFAULT]/sql_min_pool_size +# Deprecated group/name - [DATABASE]/sql_min_pool_size +#min_pool_size = 1 + +# If True, SQLite uses synchronous mode. (boolean value) +# Deprecated group/name - [DEFAULT]/sqlite_synchronous +#sqlite_synchronous = true + +# Maximum number of database connection retries during startup. Set to +# -1 to specify an infinite retry count. (integer value) +# Deprecated group/name - [DEFAULT]/sql_max_retries +# Deprecated group/name - [DATABASE]/sql_max_retries +#max_retries = 10 + +# Verbosity of SQL debugging information: 0=None, 100=Everything. +# (integer value) +# Deprecated group/name - [DEFAULT]/sql_connection_debug +#connection_debug = 0 + +# The SQL mode to be used for MySQL sessions. This option, including +# the default, overrides any server-set SQL mode. To use whatever SQL +# mode is set by the server configuration, set this to no value. +# Example: mysql_sql_mode= (string value) +#mysql_sql_mode = TRADITIONAL + +# Maximum retries in case of connection error or deadlock error before +# error is raised. Set to -1 to specify an infinite retry count. +# (integer value) +#db_max_retries = 20 + +# The back end to use for the database. (string value) +# Deprecated group/name - [DEFAULT]/db_backend +#backend = sqlalchemy # Seconds between retries of a database transaction. (integer value) #db_retry_interval = 1 @@ -269,84 +336,47 @@ # Deprecated group/name - [sql]/idle_timeout #idle_timeout = 3600 -# If set, use this value for pool_timeout with SQLAlchemy. (integer -# value) -# Deprecated group/name - [DATABASE]/sqlalchemy_pool_timeout -#pool_timeout = - -# If True, SQLite uses synchronous mode. (boolean value) -# Deprecated group/name - [DEFAULT]/sqlite_synchronous -#sqlite_synchronous = true - -# If db_inc_retry_interval is set, the maximum seconds between retries -# of a database operation. (integer value) -#db_max_retry_interval = 10 - -# Enable the experimental use of database reconnect on connection -# lost. (boolean value) -#use_db_reconnect = false - -# Interval between retries of opening a SQL connection. (integer -# value) -# Deprecated group/name - [DEFAULT]/sql_retry_interval -# Deprecated group/name - [DATABASE]/reconnect_interval -#retry_interval = 10 +# Add Python stack traces to SQL as comment strings. (boolean value) +# Deprecated group/name - [DEFAULT]/sql_connection_trace +#connection_trace = false # The file name to use with SQLite. (string value) # Deprecated group/name - [DEFAULT]/sqlite_db #sqlite_db = oslo.sqlite -# Maximum retries in case of connection error or deadlock error before -# error is raised. Set to -1 to specify an infinite retry count. -# (integer value) -#db_max_retries = 20 +# The SQLAlchemy connection string to use to connect to the database. +# (string value) +# Deprecated group/name - [DEFAULT]/sql_connection +# Deprecated group/name - [DATABASE]/sql_connection +# Deprecated group/name - [sql]/connection +#connection = # If True, increases the interval between retries of a database # operation up to db_max_retry_interval. (boolean value) #db_inc_retry_interval = true +# If db_inc_retry_interval is set, the maximum seconds between retries +# of a database operation. (integer value) +#db_max_retry_interval = 10 + +# If set, use this value for pool_timeout with SQLAlchemy. (integer +# value) +# Deprecated group/name - [DATABASE]/sqlalchemy_pool_timeout +#pool_timeout = + # The SQLAlchemy connection string to use to connect to the slave # database. (string value) #slave_connection = -# Maximum number of SQL connections to keep open in a pool. (integer -# value) -# Deprecated group/name - [DEFAULT]/sql_max_pool_size -# Deprecated group/name - [DATABASE]/sql_max_pool_size -#max_pool_size = -# Maximum number of database connection retries during startup. Set to -# -1 to specify an infinite retry count. (integer value) -# Deprecated group/name - [DEFAULT]/sql_max_retries -# Deprecated group/name - [DATABASE]/sql_max_retries -#max_retries = 10 +[glance_client] -# Minimum number of SQL connections to keep open in a pool. (integer -# value) -# Deprecated group/name - [DEFAULT]/sql_min_pool_size -# Deprecated group/name - [DATABASE]/sql_min_pool_size -#min_pool_size = 1 +# +# From watcher +# -# Verbosity of SQL debugging information: 0=None, 100=Everything. -# (integer value) -# Deprecated group/name - [DEFAULT]/sql_connection_debug -#connection_debug = 0 - -# If set, use this value for max_overflow with SQLAlchemy. (integer -# value) -# Deprecated group/name - [DEFAULT]/sql_max_overflow -# Deprecated group/name - [DATABASE]/sqlalchemy_max_overflow -#max_overflow = - -# The SQL mode to be used for MySQL sessions. This option, including -# the default, overrides any server-set SQL mode. To use whatever SQL -# mode is set by the server configuration, set this to no value. -# Example: mysql_sql_mode= (string value) -#mysql_sql_mode = TRADITIONAL - -# The back end to use for the database. (string value) -# Deprecated group/name - [DEFAULT]/db_backend -#backend = sqlalchemy +# Version of Glance API to use in glanceclient. (string value) +#api_version = 2 [keystone_authtoken] @@ -355,27 +385,11 @@ # From keystonemiddleware.auth_token # -# A PEM encoded Certificate Authority to use when verifying HTTPs -# connections. Defaults to system CAs. (string value) -#cafile = +# The region in which the identity server can be found. (string value) +#region_name = -# (Optional) Indicate whether to set the X-Service-Catalog header. If -# False, middleware will not ask for service catalog on token -# validation and will not set the X-Service-Catalog header. (boolean -# value) -#include_service_catalog = true - -# (Optional) If defined, indicate whether token data should be -# authenticated or authenticated and encrypted. If MAC, token data is -# authenticated (with HMAC) in the cache. If ENCRYPT, token data is -# encrypted and authenticated in the cache. If the value is not one of -# these options or empty, auth_token will raise an exception on -# initialization. (string value) -# Allowed values: None, MAC, ENCRYPT -#memcache_security_strategy = None - -# Verify HTTPS connections. (boolean value) -#insecure = false +# Directory used to cache files related to PKI tokens. (string value) +#signing_dir = # Used to control the use and type of token binding. Can be set to: # "disabled" to not check token binding. "permissive" (default) to @@ -386,21 +400,20 @@ # binding method that must be present in tokens. (string value) #enforce_token_bind = permissive -# The region in which the identity server can be found. (string value) -#region_name = +# Optionally specify a list of memcached server(s) to use for caching. +# If left undefined, tokens will instead be cached in-process. (list +# value) +# Deprecated group/name - [DEFAULT]/memcache_servers +#memcached_servers = + +# Env key for the swift cache. (string value) +#cache = # If true, the revocation list will be checked for cached tokens. This # requires that PKI tokens are configured on the identity server. # (boolean value) #check_revocations_for_cached = false -# Request timeout value for communicating with Identity API server. -# (integer value) -#http_connect_timeout = - -# Directory used to cache files related to PKI tokens. (string value) -#signing_dir = - # Hash algorithms to use for hashing PKI tokens. This may be a single # algorithm or multiple. The algorithms are those supported by Python # standard hashlib.new(). The hashes will be tried in the order given, @@ -412,58 +425,48 @@ # value) #hash_algorithms = md5 -# Optionally specify a list of memcached server(s) to use for caching. -# If left undefined, tokens will instead be cached in-process. (list -# value) -# Deprecated group/name - [DEFAULT]/memcache_servers -#memcached_servers = - -# Required if identity server requires client certificate (string -# value) -#certfile = - -# Authentication type to load (unknown value) -# Deprecated group/name - [DEFAULT]/auth_plugin -#auth_type = - -# Config Section from which to load plugin specific options (unknown -# value) -#auth_section = - # In order to prevent excessive effort spent validating tokens, the # middleware caches previously-seen tokens for a configurable duration # (in seconds). Set to -1 to disable caching completely. (integer # value) #token_cache_time = 300 -# API version of the admin Identity API endpoint. (string value) -#auth_version = - # Determines the frequency at which the list of revoked tokens is # retrieved from the Identity service (in seconds). A high number of # revocation events combined with a low cache duration may # significantly reduce performance. (integer value) #revocation_cache_time = 10 -# Complete public Identity API endpoint. (string value) -#auth_uri = +# Authentication type to load (unknown value) +# Deprecated group/name - [DEFAULT]/auth_plugin +#auth_type = -# (Optional) Socket timeout in seconds for communicating with a -# memcached server. (integer value) -#memcache_pool_socket_timeout = 3 +# (Optional) If defined, indicate whether token data should be +# authenticated or authenticated and encrypted. If MAC, token data is +# authenticated (with HMAC) in the cache. If ENCRYPT, token data is +# encrypted and authenticated in the cache. If the value is not one of +# these options or empty, auth_token will raise an exception on +# initialization. (string value) +# Allowed values: None, MAC, ENCRYPT +#memcache_security_strategy = None + +# (Optional) Indicate whether to set the X-Service-Catalog header. If +# False, middleware will not ask for service catalog on token +# validation and will not set the X-Service-Catalog header. (boolean +# value) +#include_service_catalog = true # (Optional, mandatory if memcache_security_strategy is defined) This # string is used for key derivation. (string value) #memcache_secret_key = -# Do not handle authorization requests within the middleware, but -# delegate the authorization decision to downstream WSGI components. -# (boolean value) -#delay_auth_decision = false +# Config Section from which to load plugin specific options (unknown +# value) +#auth_section = -# (Optional) Use the advanced (eventlet safe) memcached client pool. -# The advanced pool will only work under python 2.x. (boolean value) -#memcache_use_advanced_pool = false +# (Optional) Number of seconds memcached server is considered dead +# before it is tried again. (integer value) +#memcache_pool_dead_retry = 300 # (Optional) Maximum total number of open connections to every # memcached server. (integer value) @@ -473,24 +476,51 @@ # Identity API Server. (integer value) #http_request_max_retries = 3 -# (Optional) Number of seconds memcached server is considered dead -# before it is tried again. (integer value) -#memcache_pool_dead_retry = 300 +# Request timeout value for communicating with Identity API server. +# (integer value) +#http_connect_timeout = + +# (Optional) Socket timeout in seconds for communicating with a +# memcached server. (integer value) +#memcache_pool_socket_timeout = 3 + +# Required if identity server requires client certificate (string +# value) +#certfile = # (Optional) Number of seconds a connection to memcached is held # unused in the pool before it is closed. (integer value) #memcache_pool_unused_timeout = 60 +# Required if identity server requires client certificate (string +# value) +#keyfile = + +# Do not handle authorization requests within the middleware, but +# delegate the authorization decision to downstream WSGI components. +# (boolean value) +#delay_auth_decision = false + # (Optional) Number of seconds that an operation will wait to get a # memcached client connection from the pool. (integer value) #memcache_pool_conn_get_timeout = 10 -# Env key for the swift cache. (string value) -#cache = +# Complete public Identity API endpoint. (string value) +#auth_uri = -# Required if identity server requires client certificate (string -# value) -#keyfile = +# A PEM encoded Certificate Authority to use when verifying HTTPs +# connections. Defaults to system CAs. (string value) +#cafile = + +# API version of the admin Identity API endpoint. (string value) +#auth_version = + +# (Optional) Use the advanced (eventlet safe) memcached client pool. +# The advanced pool will only work under python 2.x. (boolean value) +#memcache_use_advanced_pool = false + +# Verify HTTPS connections. (boolean value) +#insecure = false [matchmaker_redis] @@ -499,17 +529,37 @@ # From oslo.messaging # -# Password for Redis server (optional). (string value) -#password = - # Host to locate redis. (string value) #host = 127.0.0.1 # Use this port to connect to redis host. (port value) -# Minimum value: 1 +# Minimum value: 0 # Maximum value: 65535 #port = 6379 +# Password for Redis server (optional). (string value) +#password = + + +[neutron_client] + +# +# From watcher +# + +# Version of Neutron API to use in neutronclient. (string value) +#api_version = 2 + + +[nova_client] + +# +# From watcher +# + +# Version of Nova API to use in novaclient. (string value) +#api_version = 2 + [oslo_messaging_amqp] @@ -517,73 +567,73 @@ # From oslo.messaging # -# Timeout for inactive connections (in seconds) (integer value) -# Deprecated group/name - [amqp1]/idle_timeout -#idle_timeout = 0 - -# Password for message broker authentication (string value) -# Deprecated group/name - [amqp1]/password -#password = - -# Identifying certificate PEM file to present to clients (string -# value) -# Deprecated group/name - [amqp1]/ssl_cert_file -#ssl_cert_file = +# User name for message broker authentication (string value) +# Deprecated group/name - [amqp1]/username +#username = # Private key PEM file used to sign cert_file certificate (string # value) # Deprecated group/name - [amqp1]/ssl_key_file #ssl_key_file = +# address prefix used when broadcasting to all servers (string value) +# Deprecated group/name - [amqp1]/broadcast_prefix +#broadcast_prefix = broadcast + +# Debug: dump AMQP frames to stdout (boolean value) +# Deprecated group/name - [amqp1]/trace +#trace = false + +# Identifying certificate PEM file to present to clients (string +# value) +# Deprecated group/name - [amqp1]/ssl_cert_file +#ssl_cert_file = + +# Space separated list of acceptable SASL mechanisms (string value) +# Deprecated group/name - [amqp1]/sasl_mechanisms +#sasl_mechanisms = + # address prefix when sending to any server in group (string value) # Deprecated group/name - [amqp1]/group_request_prefix #group_request_prefix = unicast +# Password for decrypting ssl_key_file (if encrypted) (string value) +# Deprecated group/name - [amqp1]/ssl_key_password +#ssl_key_password = + +# Name of configuration file (without .conf suffix) (string value) +# Deprecated group/name - [amqp1]/sasl_config_name +#sasl_config_name = + +# Password for message broker authentication (string value) +# Deprecated group/name - [amqp1]/password +#password = + +# address prefix used when sending to a specific server (string value) +# Deprecated group/name - [amqp1]/server_request_prefix +#server_request_prefix = exclusive + +# Name for the AMQP container (string value) +# Deprecated group/name - [amqp1]/container_name +#container_name = + +# CA certificate PEM file to verify server certificate (string value) +# Deprecated group/name - [amqp1]/ssl_ca_file +#ssl_ca_file = + # Path to directory that contains the SASL configuration (string # value) # Deprecated group/name - [amqp1]/sasl_config_dir #sasl_config_dir = -# Debug: dump AMQP frames to stdout (boolean value) -# Deprecated group/name - [amqp1]/trace -#trace = false - -# Password for decrypting ssl_key_file (if encrypted) (string value) -# Deprecated group/name - [amqp1]/ssl_key_password -#ssl_key_password = - -# address prefix used when sending to a specific server (string value) -# Deprecated group/name - [amqp1]/server_request_prefix -#server_request_prefix = exclusive - -# Name of configuration file (without .conf suffix) (string value) -# Deprecated group/name - [amqp1]/sasl_config_name -#sasl_config_name = - -# Name for the AMQP container (string value) -# Deprecated group/name - [amqp1]/container_name -#container_name = +# Timeout for inactive connections (in seconds) (integer value) +# Deprecated group/name - [amqp1]/idle_timeout +#idle_timeout = 0 # Accept clients using either SSL or plain TCP (boolean value) # Deprecated group/name - [amqp1]/allow_insecure_clients #allow_insecure_clients = false -# CA certificate PEM file to verify server certificate (string value) -# Deprecated group/name - [amqp1]/ssl_ca_file -#ssl_ca_file = - -# User name for message broker authentication (string value) -# Deprecated group/name - [amqp1]/username -#username = - -# address prefix used when broadcasting to all servers (string value) -# Deprecated group/name - [amqp1]/broadcast_prefix -#broadcast_prefix = broadcast - -# Space separated list of acceptable SASL mechanisms (string value) -# Deprecated group/name - [amqp1]/sasl_mechanisms -#sasl_mechanisms = - [oslo_messaging_rabbit] @@ -591,56 +641,35 @@ # From oslo.messaging # -# The RabbitMQ password. (string value) -# Deprecated group/name - [DEFAULT]/rabbit_password -#rabbit_password = guest - # SSL cert file (valid only if SSL enabled). (string value) # Deprecated group/name - [DEFAULT]/kombu_ssl_certfile #kombu_ssl_certfile = -# The RabbitMQ login method. (string value) -# Deprecated group/name - [DEFAULT]/rabbit_login_method -#rabbit_login_method = AMQPLAIN - -# How long to backoff for between retries when connecting to RabbitMQ. -# (integer value) -# Deprecated group/name - [DEFAULT]/rabbit_retry_backoff -#rabbit_retry_backoff = 2 - -# SSL certification authority file (valid only if SSL enabled). -# (string value) -# Deprecated group/name - [DEFAULT]/kombu_ssl_ca_certs -#kombu_ssl_ca_certs = - -# The RabbitMQ virtual host. (string value) -# Deprecated group/name - [DEFAULT]/rabbit_virtual_host -#rabbit_virtual_host = / - -# How long to wait before reconnecting in response to an AMQP consumer -# cancel notification. (floating point value) -# Deprecated group/name - [DEFAULT]/kombu_reconnect_delay -#kombu_reconnect_delay = 1.0 - -# How frequently to retry connecting with RabbitMQ. (integer value) -#rabbit_retry_interval = 1 - -# How long to wait a missing client beforce abandoning to send it its -# replies. This value should not be longer than rpc_response_timeout. -# (integer value) -# Deprecated group/name - [DEFAULT]/kombu_reconnect_timeout -#kombu_missing_consumer_retry_timeout = 60 +# Use HA queues in RabbitMQ (x-ha-policy: all). If you change this +# option, you must wipe the RabbitMQ database. (boolean value) +# Deprecated group/name - [DEFAULT]/rabbit_ha_queues +#rabbit_ha_queues = false # Deprecated, use rpc_backend=kombu+memory or rpc_backend=fake # (boolean value) # Deprecated group/name - [DEFAULT]/fake_rabbit #fake_rabbit = false -# Determines how the next RabbitMQ node is chosen in case the one we -# are currently connected to becomes unavailable. Takes effect only if -# more than one RabbitMQ node is provided in config. (string value) -# Allowed values: round-robin, shuffle -#kombu_failover_strategy = round-robin +# Number of seconds after which the Rabbit broker is considered down +# if heartbeat's keep-alive fails (0 disable the heartbeat). +# EXPERIMENTAL (integer value) +#heartbeat_timeout_threshold = 60 + +# How long to wait before reconnecting in response to an AMQP consumer +# cancel notification. (floating point value) +# Deprecated group/name - [DEFAULT]/kombu_reconnect_delay +#kombu_reconnect_delay = 1.0 + +# How long to wait a missing client beforce abandoning to send it its +# replies. This value should not be longer than rpc_response_timeout. +# (integer value) +# Deprecated group/name - [DEFAULT]/kombu_reconnect_timeout +#kombu_missing_consumer_retry_timeout = 60 # How often times during the heartbeat_timeout_threshold we check the # heartbeat. (integer value) @@ -652,42 +681,57 @@ #rabbit_host = localhost # The RabbitMQ broker port where a single node is used. (port value) -# Minimum value: 1 +# Minimum value: 0 # Maximum value: 65535 # Deprecated group/name - [DEFAULT]/rabbit_port #rabbit_port = 5672 -# Use HA queues in RabbitMQ (x-ha-policy: all). If you change this -# option, you must wipe the RabbitMQ database. (boolean value) -# Deprecated group/name - [DEFAULT]/rabbit_ha_queues -#rabbit_ha_queues = false - # Use durable queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_durable_queues # Deprecated group/name - [DEFAULT]/rabbit_durable_queues #amqp_durable_queues = false -# Maximum number of RabbitMQ connection retries. Default is 0 -# (infinite retry count). (integer value) -# Deprecated group/name - [DEFAULT]/rabbit_max_retries -#rabbit_max_retries = 0 +# Determines how the next RabbitMQ node is chosen in case the one we +# are currently connected to becomes unavailable. Takes effect only if +# more than one RabbitMQ node is provided in config. (string value) +# Allowed values: round-robin, shuffle +#kombu_failover_strategy = round-robin # RabbitMQ HA cluster host:port pairs. (list value) # Deprecated group/name - [DEFAULT]/rabbit_hosts #rabbit_hosts = $rabbit_host:$rabbit_port +# Connect over SSL for RabbitMQ. (boolean value) +# Deprecated group/name - [DEFAULT]/rabbit_use_ssl +#rabbit_use_ssl = false + +# SSL certification authority file (valid only if SSL enabled). +# (string value) +# Deprecated group/name - [DEFAULT]/kombu_ssl_ca_certs +#kombu_ssl_ca_certs = + +# The RabbitMQ userid. (string value) +# Deprecated group/name - [DEFAULT]/rabbit_userid +#rabbit_userid = guest + +# The RabbitMQ password. (string value) +# Deprecated group/name - [DEFAULT]/rabbit_password +#rabbit_password = guest + +# The RabbitMQ login method. (string value) +# Deprecated group/name - [DEFAULT]/rabbit_login_method +#rabbit_login_method = AMQPLAIN + +# The RabbitMQ virtual host. (string value) +# Deprecated group/name - [DEFAULT]/rabbit_virtual_host +#rabbit_virtual_host = / + # Auto-delete queues in AMQP. (boolean value) # Deprecated group/name - [DEFAULT]/amqp_auto_delete #amqp_auto_delete = false -# Number of seconds after which the Rabbit broker is considered down -# if heartbeat's keep-alive fails (0 disable the heartbeat). -# EXPERIMENTAL (integer value) -#heartbeat_timeout_threshold = 60 - -# Connect over SSL for RabbitMQ. (boolean value) -# Deprecated group/name - [DEFAULT]/rabbit_use_ssl -#rabbit_use_ssl = false +# How frequently to retry connecting with RabbitMQ. (integer value) +#rabbit_retry_interval = 1 # SSL version to use (valid only if SSL enabled). Valid values are # TLSv1 and SSLv23. SSLv2, SSLv3, TLSv1_1, and TLSv1_2 may be @@ -695,14 +739,20 @@ # Deprecated group/name - [DEFAULT]/kombu_ssl_version #kombu_ssl_version = -# The RabbitMQ userid. (string value) -# Deprecated group/name - [DEFAULT]/rabbit_userid -#rabbit_userid = guest +# How long to backoff for between retries when connecting to RabbitMQ. +# (integer value) +# Deprecated group/name - [DEFAULT]/rabbit_retry_backoff +#rabbit_retry_backoff = 2 # SSL key file (valid only if SSL enabled). (string value) # Deprecated group/name - [DEFAULT]/kombu_ssl_keyfile #kombu_ssl_keyfile = +# Maximum number of RabbitMQ connection retries. Default is 0 +# (infinite retry count). (integer value) +# Deprecated group/name - [DEFAULT]/rabbit_max_retries +#rabbit_max_retries = 0 + [watcher_applier] @@ -710,48 +760,131 @@ # From watcher # -# Number of workers for applier, default value is 1. (integer value) -# Minimum value: 1 -#workers = 1 - -# Select the engine to use to execute the workflow (string value) -#workflow_engine = taskflow - # The topic name used forcontrol events, this topic used for rpc call # (string value) #topic_control = watcher.applier.control +# Number of workers for applier, default value is 1. (integer value) +# Minimum value: 1 +#workers = 1 + # The identifier used by watcher module on the message broker (string # value) #publisher_id = watcher.applier.api +# Select the engine to use to execute the workflow (string value) +#workflow_engine = taskflow + # The topic name used for status events, this topic is used so as to # notifythe others components of the system (string value) #topic_status = watcher.applier.status +[watcher_clients_auth] + +# +# From watcher +# + +# Authentication URL (unknown value) +#auth_url = + +# Domain ID to scope to (unknown value) +#domain_id = + +# Domain name to scope to (unknown value) +#domain_name = + +# Domain ID containing project (unknown value) +#project_domain_id = + +# Project ID to scope to (unknown value) +# Deprecated group/name - [DEFAULT]/tenant-id +#project_id = + +# Project name to scope to (unknown value) +# Deprecated group/name - [DEFAULT]/tenant-name +#project_name = + +# PEM encoded client certificate cert file (string value) +#certfile = + +# PEM encoded client certificate key file (string value) +#keyfile = + +# Domain name containing project (unknown value) +#project_domain_name = + +# Trust ID (unknown value) +#trust_id = + +# Optional domain ID to use with v3 and v2 parameters. It will be used +# for both the user and project domain in v3 and ignored in v2 +# authentication. (unknown value) +#default_domain_id = + +# Timeout value for http requests (integer value) +#timeout = + +# Optional domain name to use with v3 API and v2 parameters. It will +# be used for both the user and project domain in v3 and ignored in v2 +# authentication. (unknown value) +#default_domain_name = + +# User id (unknown value) +#user_id = + +# Username (unknown value) +# Deprecated group/name - [DEFAULT]/username +#username = + +# Verify HTTPS connections. (boolean value) +#insecure = false + +# User's domain name (unknown value) +#user_domain_name = + +# Authentication type to load (unknown value) +# Deprecated group/name - [DEFAULT]/auth_plugin +#auth_type = + +# Config Section from which to load plugin specific options (unknown +# value) +#auth_section = + +# User's password (unknown value) +#password = + +# User's domain id (unknown value) +#user_domain_id = + +# PEM encoded Certificate Authority to use when verifying HTTPs +# connections. (string value) +#cafile = + + [watcher_decision_engine] # # From watcher # -# The identifier used by watcher module on the message broker (string -# value) -#publisher_id = watcher.decision.api +# The topic name used for status events, this topic is used so as to +# notifythe others components of the system (string value) +#topic_status = watcher.decision.status # The topic name used forcontrol events, this topic used for rpc call # (string value) #topic_control = watcher.decision.control +# The identifier used by watcher module on the message broker (string +# value) +#publisher_id = watcher.decision.api + # The maximum number of threads that can be used to execute strategies # (integer value) #max_workers = 2 -# The topic name used for status events, this topic is used so as to -# notifythe others components of the system (string value) -#topic_status = watcher.decision.status - [watcher_goals] diff --git a/requirements.txt b/requirements.txt index 2e66858f0..bcb586e28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ enum34;python_version=='2.7' or python_version=='2.6' jsonpatch>=1.1 +keystoneauth1>=2.1.0 keystonemiddleware>=2.0.0,!=2.4.0 oslo.config>=2.3.0 # Apache-2.0 oslo.db>=2.4.1 # Apache-2.0 diff --git a/watcher/applier/action_plan/default.py b/watcher/applier/action_plan/default.py index b6711f884..12f9f8d19 100644 --- a/watcher/applier/action_plan/default.py +++ b/watcher/applier/action_plan/default.py @@ -52,7 +52,7 @@ class DefaultActionPlanHandler(base.BaseActionPlanHandler): self.notify(self.action_plan_uuid, event_types.EventTypes.LAUNCH_ACTION_PLAN, ap_objects.State.ONGOING) - applier = default.DefaultApplier(self.applier_manager, self.ctx) + applier = default.DefaultApplier(self.ctx, self.applier_manager) result = applier.execute(self.action_plan_uuid) except Exception as e: LOG.exception(e) diff --git a/watcher/applier/actions/base.py b/watcher/applier/actions/base.py index facf47911..e70020c65 100644 --- a/watcher/applier/actions/base.py +++ b/watcher/applier/actions/base.py @@ -22,12 +22,22 @@ import abc import six +from watcher.common import clients + @six.add_metaclass(abc.ABCMeta) class BaseAction(object): - def __init__(self): + def __init__(self, osc=None): + """:param osc: an OpenStackClients instance""" self._input_parameters = {} self._applies_to = "" + self._osc = osc + + @property + def osc(self): + if not self._osc: + self._osc = clients.OpenStackClients() + return self._osc @property def input_parameters(self): diff --git a/watcher/applier/actions/change_nova_service_state.py b/watcher/applier/actions/change_nova_service_state.py index b7bb40524..ecb61edb0 100644 --- a/watcher/applier/actions/change_nova_service_state.py +++ b/watcher/applier/actions/change_nova_service_state.py @@ -21,8 +21,7 @@ from watcher._i18n import _ from watcher.applier.actions import base from watcher.common import exception -from watcher.common import keystone as kclient -from watcher.common import nova as nclient +from watcher.common import nova_helper from watcher.decision_engine.model import hypervisor_state as hstate @@ -57,13 +56,11 @@ class ChangeNovaServiceState(base.BaseAction): raise exception.IllegalArgumentException( message=_("The target state is not defined")) - keystone = kclient.KeystoneClient() - wrapper = nclient.NovaClient(keystone.get_credentials(), - session=keystone.get_session()) + nova = nova_helper.NovaHelper(osc=self.osc) if state is True: - return wrapper.enable_service_nova_compute(self.host) + return nova.enable_service_nova_compute(self.host) else: - return wrapper.disable_service_nova_compute(self.host) + return nova.disable_service_nova_compute(self.host) def precondition(self): pass diff --git a/watcher/applier/actions/factory.py b/watcher/applier/actions/factory.py index d21600adc..24e0ecaba 100644 --- a/watcher/applier/actions/factory.py +++ b/watcher/applier/actions/factory.py @@ -28,9 +28,10 @@ class ActionFactory(object): def __init__(self): self.action_loader = default.DefaultActionLoader() - def make_action(self, object_action): + def make_action(self, object_action, osc=None): LOG.debug("Creating instance of %s", object_action.action_type) - loaded_action = self.action_loader.load(name=object_action.action_type) + loaded_action = self.action_loader.load(name=object_action.action_type, + osc=osc) loaded_action.input_parameters = object_action.input_parameters loaded_action.applies_to = object_action.applies_to return loaded_action diff --git a/watcher/applier/actions/migration.py b/watcher/applier/actions/migration.py index e23632937..f25dc0aa3 100644 --- a/watcher/applier/actions/migration.py +++ b/watcher/applier/actions/migration.py @@ -21,8 +21,7 @@ from oslo_log import log from watcher.applier.actions import base from watcher.common import exception -from watcher.common import keystone as kclient -from watcher.common import nova as nclient +from watcher.common import nova_helper LOG = log.getLogger(__name__) @@ -45,15 +44,13 @@ class Migrate(base.BaseAction): return self.input_parameters.get('src_hypervisor') def migrate(self, destination): - keystone = kclient.KeystoneClient() - wrapper = nclient.NovaClient(keystone.get_credentials(), - session=keystone.get_session()) - LOG.debug("Migrate instance %s to %s ", self.instance_uuid, + nova = nova_helper.NovaHelper(osc=self.osc) + LOG.debug("Migrate instance %s to %s", self.instance_uuid, destination) - instance = wrapper.find_instance(self.instance_uuid) + instance = nova.find_instance(self.instance_uuid) if instance: if self.migration_type == 'live': - return wrapper.live_migrate_instance( + return nova.live_migrate_instance( instance_id=self.instance_uuid, dest_hostname=destination) else: raise exception.Invalid( diff --git a/watcher/applier/default.py b/watcher/applier/default.py index d2b4529c5..ac1c7810c 100644 --- a/watcher/applier/default.py +++ b/watcher/applier/default.py @@ -28,7 +28,7 @@ CONF = cfg.CONF class DefaultApplier(base.BaseApplier): - def __init__(self, applier_manager, context): + def __init__(self, context, applier_manager): super(DefaultApplier, self).__init__() self._applier_manager = applier_manager self._loader = default.DefaultWorkFlowEngineLoader() @@ -48,9 +48,10 @@ class DefaultApplier(base.BaseApplier): if self._engine is None: selected_workflow_engine = CONF.watcher_applier.workflow_engine LOG.debug("Loading workflow engine %s ", selected_workflow_engine) - self._engine = self._loader.load(name=selected_workflow_engine) - self._engine.context = self.context - self._engine.applier_manager = self.applier_manager + self._engine = self._loader.load( + name=selected_workflow_engine, + context=self.context, + applier_manager=self.applier_manager) return self._engine def execute(self, action_plan_uuid): diff --git a/watcher/applier/workflow_engine/base.py b/watcher/applier/workflow_engine/base.py index 4cb95146d..255c898cf 100644 --- a/watcher/applier/workflow_engine/base.py +++ b/watcher/applier/workflow_engine/base.py @@ -22,33 +22,33 @@ import six from watcher.applier.actions import factory from watcher.applier.messaging import event_types +from watcher.common import clients from watcher.common.messaging.events import event from watcher import objects @six.add_metaclass(abc.ABCMeta) class BaseWorkFlowEngine(object): - def __init__(self): - self._applier_manager = None - self._context = None + def __init__(self, context=None, applier_manager=None): + self._context = context + self._applier_manager = applier_manager self._action_factory = factory.ActionFactory() + self._osc = None @property def context(self): return self._context - @context.setter - def context(self, c): - self._context = c + @property + def osc(self): + if not self._osc: + self._osc = clients.OpenStackClients() + return self._osc @property def applier_manager(self): return self._applier_manager - @applier_manager.setter - def applier_manager(self, a): - self._applier_manager = a - @property def action_factory(self): return self._action_factory diff --git a/watcher/applier/workflow_engine/default.py b/watcher/applier/workflow_engine/default.py index 6124ced37..ba653e7f2 100644 --- a/watcher/applier/workflow_engine/default.py +++ b/watcher/applier/workflow_engine/default.py @@ -89,7 +89,9 @@ class TaskFlowActionContainer(task.Task): @property def action(self): if self.loaded_action is None: - action = self.engine.action_factory.make_action(self._db_action) + action = self.engine.action_factory.make_action( + self._db_action, + osc=self._engine.osc) self.loaded_action = action return self.loaded_action diff --git a/watcher/common/ceilometer.py b/watcher/common/ceilometer_helper.py similarity index 80% rename from watcher/common/ceilometer.py rename to watcher/common/ceilometer_helper.py index 703733df4..4e155123f 100644 --- a/watcher/common/ceilometer.py +++ b/watcher/common/ceilometer_helper.py @@ -17,30 +17,16 @@ # limitations under the License. # -from ceilometerclient import client from ceilometerclient.exc import HTTPUnauthorized -from watcher.common import keystone +from watcher.common import clients -class CeilometerClient(object): - def __init__(self, api_version='2'): - self._cmclient = None - self._api_version = api_version - - @property - def cmclient(self): - """Initialization of Ceilometer client.""" - if not self._cmclient: - ksclient = keystone.KeystoneClient() - creds = ksclient.get_credentials() - endpoint = ksclient.get_endpoint( - service_type='metering', - endpoint_type='publicURL') - self._cmclient = client.get_client(self._api_version, - ceilometer_url=endpoint, - **creds) - return self._cmclient +class CeilometerHelper(object): + def __init__(self, osc=None): + """:param osc: an OpenStackClients instance""" + self.osc = osc if osc else clients.OpenStackClients() + self.ceilometer = self.osc.ceilometer() def build_query(self, user_id=None, tenant_id=None, resource_id=None, user_ids=None, tenant_ids=None, resource_ids=None): @@ -83,20 +69,21 @@ class CeilometerClient(object): try: return f(*args, **kargs) except HTTPUnauthorized: - self.reset_client() + self.osc.reset_clients() + self.ceilometer = self.osc.ceilometer() return f(*args, **kargs) except Exception: raise def query_sample(self, meter_name, query, limit=1): - return self.query_retry(f=self.cmclient.samples.list, + return self.query_retry(f=self.ceilometer.samples.list, meter_name=meter_name, limit=limit, q=query) def statistic_list(self, meter_name, query=None, period=None): """List of statistics.""" - statistics = self.cmclient.statistics.list( + statistics = self.ceilometer.statistics.list( meter_name=meter_name, q=query, period=period) @@ -104,7 +91,8 @@ class CeilometerClient(object): def meter_list(self, query=None): """List the user's meters.""" - meters = self.query_retry(f=self.cmclient.meters.list, query=query) + meters = self.query_retry(f=self.ceilometer.meters.list, + query=query) return meters def statistic_aggregation(self, @@ -125,7 +113,7 @@ class CeilometerClient(object): """ query = self.build_query(resource_id=resource_id) - statistic = self.query_retry(f=self.cmclient.statistics.list, + statistic = self.query_retry(f=self.ceilometer.statistics.list, meter_name=meter_name, q=query, period=period, @@ -156,6 +144,3 @@ class CeilometerClient(object): return samples[-1]._info['counter_volume'] else: return False - - def reset_client(self): - self._cmclient = None diff --git a/watcher/common/clients.py b/watcher/common/clients.py new file mode 100644 index 000000000..1b0189ed8 --- /dev/null +++ b/watcher/common/clients.py @@ -0,0 +1,158 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from ceilometerclient import client as ceclient +from cinderclient import client as ciclient +from glanceclient import client as glclient +from keystoneauth1 import loading as ka_loading +from keystoneclient import client as keyclient +from neutronclient.neutron import client as netclient +from novaclient import client as nvclient +from oslo_config import cfg + +from watcher._i18n import _ +from watcher.common import exception + + +NOVA_CLIENT_OPTS = [ + cfg.StrOpt('api_version', + default='2', + help=_('Version of Nova API to use in novaclient.'))] + +GLANCE_CLIENT_OPTS = [ + cfg.StrOpt('api_version', + default='2', + help=_('Version of Glance API to use in glanceclient.'))] + +CINDER_CLIENT_OPTS = [ + cfg.StrOpt('api_version', + default='2', + help=_('Version of Cinder API to use in cinderclient.'))] + +CEILOMETER_CLIENT_OPTS = [ + cfg.StrOpt('api_version', + default='2', + help=_('Version of Ceilometer API to use in ' + 'ceilometerclient.'))] + +NEUTRON_CLIENT_OPTS = [ + cfg.StrOpt('api_version', + default='2', + help=_('Version of Neutron API to use in neutronclient.'))] + +cfg.CONF.register_opts(NOVA_CLIENT_OPTS, group='nova_client') +cfg.CONF.register_opts(GLANCE_CLIENT_OPTS, group='glance_client') +cfg.CONF.register_opts(CINDER_CLIENT_OPTS, group='cinder_client') +cfg.CONF.register_opts(CEILOMETER_CLIENT_OPTS, group='ceilometer_client') +cfg.CONF.register_opts(NEUTRON_CLIENT_OPTS, group='neutron_client') + +_CLIENTS_AUTH_GROUP = 'watcher_clients_auth' + +ka_loading.register_auth_conf_options(cfg.CONF, _CLIENTS_AUTH_GROUP) +ka_loading.register_session_conf_options(cfg.CONF, _CLIENTS_AUTH_GROUP) + + +class OpenStackClients(object): + """Convenience class to create and cache client instances.""" + + def __init__(self): + self.reset_clients() + + def reset_clients(self): + self._session = None + self._keystone = None + self._nova = None + self._glance = None + self._cinder = None + self._ceilometer = None + self._neutron = None + + def _get_keystone_session(self): + auth = ka_loading.load_auth_from_conf_options(cfg.CONF, + _CLIENTS_AUTH_GROUP) + sess = ka_loading.load_session_from_conf_options(cfg.CONF, + _CLIENTS_AUTH_GROUP, + auth=auth) + return sess + + @property + def auth_url(self): + return self.keystone().auth_url + + @property + def session(self): + if not self._session: + self._session = self._get_keystone_session() + return self._session + + def _get_client_option(self, client, option): + return getattr(getattr(cfg.CONF, '%s_client' % client), option) + + @exception.wrap_keystone_exception + def keystone(self): + if not self._keystone: + self._keystone = keyclient.Client(session=self.session) + + return self._keystone + + @exception.wrap_keystone_exception + def nova(self): + if self._nova: + return self._nova + + novaclient_version = self._get_client_option('nova', 'api_version') + self._nova = nvclient.Client(novaclient_version, + session=self.session) + return self._nova + + @exception.wrap_keystone_exception + def glance(self): + if self._glance: + return self._glance + + glanceclient_version = self._get_client_option('glance', 'api_version') + self._glance = glclient.Client(glanceclient_version, + session=self.session) + return self._glance + + @exception.wrap_keystone_exception + def cinder(self): + if self._cinder: + return self._cinder + + cinderclient_version = self._get_client_option('cinder', 'api_version') + self._cinder = ciclient.Client(cinderclient_version, + session=self.session) + return self._cinder + + @exception.wrap_keystone_exception + def ceilometer(self): + if self._ceilometer: + return self._ceilometer + + ceilometerclient_version = self._get_client_option('ceilometer', + 'api_version') + self._ceilometer = ceclient.Client(ceilometerclient_version, + session=self.session) + return self._ceilometer + + @exception.wrap_keystone_exception + def neutron(self): + if self._neutron: + return self._neutron + + neutronclient_version = self._get_client_option('neutron', + 'api_version') + self._neutron = netclient.Client(neutronclient_version, + session=self.session) + self._neutron.format = 'json' + return self._neutron diff --git a/watcher/common/exception.py b/watcher/common/exception.py index e2c9d73df..7043ccb46 100644 --- a/watcher/common/exception.py +++ b/watcher/common/exception.py @@ -22,6 +22,10 @@ SHOULD include dedicated exception logging. """ +import functools +import sys + +from keystoneclient import exceptions as keystone_exceptions from oslo_config import cfg from oslo_log import log as logging import six @@ -40,6 +44,23 @@ CONF = cfg.CONF CONF.register_opts(exc_log_opts) +def wrap_keystone_exception(func): + """Wrap keystone exceptions and throw Watcher specific exceptions.""" + @functools.wraps(func) + def wrapped(*args, **kw): + try: + return func(*args, **kw) + except keystone_exceptions.AuthorizationFailure: + raise AuthorizationFailure( + client=func.__name__, reason=sys.exc_info()[1]) + except keystone_exceptions.ClientException: + raise AuthorizationFailure( + client=func.__name__, + reason=(_('Unexpected keystone client error occurred: %s') + % sys.exc_info()[1])) + return wrapped + + class WatcherException(Exception): """Base Watcher Exception @@ -226,6 +247,10 @@ class NoDataFound(WatcherException): msg_fmt = _('No rows were returned') +class AuthorizationFailure(WatcherException): + msg_fmt = _('%(client)s connection failed. Reason: %(reason)s') + + class KeystoneFailure(WatcherException): msg_fmt = _("'Keystone API endpoint is missing''") diff --git a/watcher/common/keystone.py b/watcher/common/keystone.py deleted file mode 100644 index c58ebbd44..000000000 --- a/watcher/common/keystone.py +++ /dev/null @@ -1,131 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Authors: Jean-Emile DARTOIS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from keystoneclient.auth.identity import generic -from keystoneclient import session as keystone_session -from oslo_config import cfg -from oslo_log import log -from six.moves.urllib.parse import urljoin -from six.moves.urllib.parse import urlparse - -from watcher._i18n import _ -from watcher.common import exception - - -LOG = log.getLogger(__name__) -CONF = cfg.CONF - -CONF.import_opt('admin_user', 'keystonemiddleware.auth_token', - group='keystone_authtoken') -CONF.import_opt('admin_tenant_name', 'keystonemiddleware.auth_token', - group='keystone_authtoken') -CONF.import_opt('admin_password', 'keystonemiddleware.auth_token', - group='keystone_authtoken') -CONF.import_opt('auth_uri', 'keystonemiddleware.auth_token', - group='keystone_authtoken') -CONF.import_opt('auth_version', 'keystonemiddleware.auth_token', - group='keystone_authtoken') -CONF.import_opt('insecure', 'keystonemiddleware.auth_token', - group='keystone_authtoken') - - -class KeystoneClient(object): - def __init__(self): - self._ks_client = None - self._session = None - self._auth = None - self._token = None - - def get_endpoint(self, **kwargs): - kc = self.get_ksclient() - if not kc.has_service_catalog(): - raise exception.KeystoneFailure( - _('No Keystone service catalog loaded') - ) - attr = None - filter_value = None - if kwargs.get('region_name'): - attr = 'region' - filter_value = kwargs.get('region_name') - return kc.service_catalog.url_for( - service_type=kwargs.get('service_type') or 'metering', - attr=attr, - filter_value=filter_value, - endpoint_type=kwargs.get('endpoint_type') or 'publicURL') - - def _is_apiv3(self, auth_url, auth_version): - return auth_version == 'v3.0' or '/v3' in urlparse(auth_url).path - - def get_keystone_url(self, auth_url, auth_version): - """Gives an http/https url to contact keystone.""" - - api_v3 = self._is_apiv3(auth_url, auth_version) - api_version = 'v3' if api_v3 else 'v2.0' - # NOTE(lucasagomes): Get rid of the trailing '/' otherwise urljoin() - # fails to override the version in the URL - return urljoin(auth_url.rstrip('/'), api_version) - - def get_ksclient(self, creds=None): - """Get an endpoint and auth token from Keystone.""" - auth_version = CONF.keystone_authtoken.auth_version - auth_url = CONF.keystone_authtoken.auth_uri - api_v3 = self._is_apiv3(auth_url, auth_version) - if creds is None: - ks_args = self._get_credentials(api_v3) - else: - ks_args = creds - - if api_v3: - from keystoneclient.v3 import client - else: - from keystoneclient.v2_0 import client - # generic - # ksclient = client.Client(version=api_version, - # session=session,**ks_args) - - return client.Client(**ks_args) - - def _get_credentials(self, api_v3): - if api_v3: - creds = \ - {'auth_url': CONF.keystone_authtoken.auth_uri, - 'username': CONF.keystone_authtoken.admin_user, - 'password': CONF.keystone_authtoken.admin_password, - 'project_name': CONF.keystone_authtoken.admin_tenant_name, - 'user_domain_name': "default", - 'project_domain_name': "default"} - else: - creds = \ - {'auth_url': CONF.keystone_authtoken.auth_uri, - 'username': CONF.keystone_authtoken.admin_user, - 'password': CONF.keystone_authtoken.admin_password, - 'tenant_name': CONF.keystone_authtoken.admin_tenant_name} - LOG.debug(creds) - return creds - - def get_credentials(self): - api_v3 = self._is_apiv3(CONF.keystone_authtoken.auth_uri, - CONF.keystone_authtoken.auth_version) - return self._get_credentials(api_v3) - - def get_session(self): - creds = self.get_credentials() - self._auth = generic.Password(**creds) - session = keystone_session.Session(auth=self._auth) - return session diff --git a/watcher/common/loader/default.py b/watcher/common/loader/default.py index 34dac2f53..9048548e7 100644 --- a/watcher/common/loader/default.py +++ b/watcher/common/loader/default.py @@ -31,7 +31,7 @@ class DefaultLoader(BaseLoader): super(DefaultLoader, self).__init__() self.namespace = namespace - def load(self, name): + def load(self, name, **kwargs): try: LOG.debug("Loading in namespace %s => %s ", self.namespace, name) driver_manager = DriverManager(namespace=self.namespace, @@ -41,7 +41,7 @@ class DefaultLoader(BaseLoader): LOG.exception(exc) raise exception.LoadingError(name=name) - return loaded() + return loaded(**kwargs) def list_available(self): extension_manager = ExtensionManager(namespace=self.namespace) diff --git a/watcher/common/nova.py b/watcher/common/nova_helper.py similarity index 95% rename from watcher/common/nova.py rename to watcher/common/nova_helper.py index 58fcc09d5..bb237d323 100644 --- a/watcher/common/nova.py +++ b/watcher/common/nova_helper.py @@ -23,29 +23,22 @@ import time from oslo_log import log import cinderclient.exceptions as ciexceptions -import cinderclient.v2.client as ciclient -import glanceclient.v2.client as glclient -import neutronclient.neutron.client as netclient -import novaclient.client as nvclient import novaclient.exceptions as nvexceptions -from watcher.common import keystone +from watcher.common import clients LOG = log.getLogger(__name__) -class NovaClient(object): - NOVA_CLIENT_API_VERSION = "2" +class NovaHelper(object): - def __init__(self, creds, session): - self.user = creds['username'] - self.session = session - self.neutron = None - self.cinder = None - self.nova = nvclient.Client(self.NOVA_CLIENT_API_VERSION, - session=session) - self.keystone = keystone.KeystoneClient().get_ksclient(creds) - self.glance = None + def __init__(self, osc=None): + """:param osc: an OpenStackClients instance""" + self.osc = osc if osc else clients.OpenStackClients() + self.neutron = self.osc.neutron() + self.cinder = self.osc.cinder() + self.nova = self.osc.nova() + self.glance = self.osc.glance() def get_hypervisors_list(self): return self.nova.hypervisors.list() @@ -180,9 +173,6 @@ class NovaClient(object): volume_id = attached_volume['id'] try: - if self.cinder is None: - self.cinder = ciclient.Client('2', - session=self.session) volume = self.cinder.volumes.get(volume_id) attachments_list = getattr(volume, "attachments") @@ -446,13 +436,6 @@ class NovaClient(object): :param metadata: a dictionary containing the list of key-value pairs to associate to the image as metadata. """ - if self.glance is None: - glance_endpoint = self.keystone. \ - service_catalog.url_for(service_type='image', - endpoint_type='publicURL') - self.glance = glclient.Client(glance_endpoint, - token=self.keystone.auth_token) - LOG.debug( "Trying to create an image from instance %s ..." % instance_id) @@ -676,10 +659,6 @@ class NovaClient(object): def get_network_id_from_name(self, net_name="private"): """This method returns the unique id of the provided network name""" - if self.neutron is None: - self.neutron = netclient.Client('2.0', session=self.session) - self.neutron.format = 'json' - networks = self.neutron.list_networks(name=net_name) # LOG.debug(networks) diff --git a/watcher/decision_engine/strategy/context/default.py b/watcher/decision_engine/strategy/context/default.py index 2310cbbea..5c4f8e622 100644 --- a/watcher/decision_engine/strategy/context/default.py +++ b/watcher/decision_engine/strategy/context/default.py @@ -15,6 +15,7 @@ # limitations under the License. from oslo_log import log +from watcher.common import clients from watcher.decision_engine.strategy.context.base import BaseStrategyContext from watcher.decision_engine.strategy.selection.default import \ DefaultStrategySelector @@ -47,15 +48,17 @@ class DefaultStrategyContext(BaseStrategyContext): audit_template = objects.\ AuditTemplate.get_by_id(request_context, audit.audit_template_id) + osc = clients.OpenStackClients() + # todo(jed) retrieve in audit_template parameters (threshold,...) # todo(jed) create ActionPlan - collector_manager = self.collector.get_cluster_model_collector() + collector_manager = self.collector.get_cluster_model_collector(osc=osc) # todo(jed) remove call to get_latest_cluster_data_model cluster_data_model = collector_manager.get_latest_cluster_data_model() - selected_strategy = self.strategy_selector. \ - define_from_goal(audit_template.goal) + selected_strategy = self.strategy_selector.define_from_goal( + audit_template.goal, osc=osc) # todo(jed) add parameters and remove cluster_data_model return selected_strategy.execute(cluster_data_model) diff --git a/watcher/decision_engine/strategy/selection/default.py b/watcher/decision_engine/strategy/selection/default.py index 28e6984f8..d81a03891 100644 --- a/watcher/decision_engine/strategy/selection/default.py +++ b/watcher/decision_engine/strategy/selection/default.py @@ -48,11 +48,12 @@ class DefaultStrategySelector(base.BaseSelector): super(DefaultStrategySelector, self).__init__() self.strategy_loader = default.DefaultStrategyLoader() - def define_from_goal(self, goal_name): + def define_from_goal(self, goal_name, osc=None): + """:param osc: an OpenStackClients instance""" strategy_to_load = None try: strategy_to_load = CONF.watcher_goals.goals[goal_name] - return self.strategy_loader.load(strategy_to_load) + return self.strategy_loader.load(strategy_to_load, osc=osc) except KeyError as exc: LOG.exception(exc) raise exception.WatcherException( diff --git a/watcher/decision_engine/strategy/strategies/base.py b/watcher/decision_engine/strategy/strategies/base.py index c8917d490..d43e4c5b0 100644 --- a/watcher/decision_engine/strategy/strategies/base.py +++ b/watcher/decision_engine/strategy/strategies/base.py @@ -33,10 +33,11 @@ provided as well. """ import abc + from oslo_log import log import six - +from watcher.common import clients from watcher.decision_engine.solution.default import DefaultSolution from watcher.decision_engine.strategy.common.level import StrategyLevel @@ -51,7 +52,8 @@ class BaseStrategy(object): Solution for a given Goal. """ - def __init__(self, name=None, description=None): + def __init__(self, name=None, description=None, osc=None): + """:param osc: an OpenStackClients instance""" self._name = name self.description = description # default strategy level @@ -59,6 +61,7 @@ class BaseStrategy(object): self._cluster_state_collector = None # the solution given by the strategy self._solution = DefaultSolution() + self._osc = osc @abc.abstractmethod def execute(self, model): @@ -70,6 +73,12 @@ class BaseStrategy(object): :rtype: :class:`watcher.decision_engine.solution.base.BaseSolution` """ + @property + def osc(self): + if not self._osc: + self._osc = clients.OpenStackClients() + return self._osc + @property def solution(self): return self._solution diff --git a/watcher/decision_engine/strategy/strategies/basic_consolidation.py b/watcher/decision_engine/strategy/strategies/basic_consolidation.py index f059034fb..2a8cfdfc5 100644 --- a/watcher/decision_engine/strategy/strategies/basic_consolidation.py +++ b/watcher/decision_engine/strategy/strategies/basic_consolidation.py @@ -41,7 +41,8 @@ class BasicConsolidation(BaseStrategy): MIGRATION = "migrate" CHANGE_NOVA_SERVICE_STATE = "change_nova_service_state" - def __init__(self, name=DEFAULT_NAME, description=DEFAULT_DESCRIPTION): + def __init__(self, name=DEFAULT_NAME, description=DEFAULT_DESCRIPTION, + osc=None): """Basic offline Consolidation using live migration The basic consolidation algorithm has several limitations. @@ -68,8 +69,9 @@ class BasicConsolidation(BaseStrategy): :param name: the name of the strategy :param description: a description of the strategy + :param osc: an OpenStackClients object """ - super(BasicConsolidation, self).__init__(name, description) + super(BasicConsolidation, self).__init__(name, description, osc) # set default value for the number of released nodes self.number_of_released_nodes = 0 @@ -102,7 +104,7 @@ class BasicConsolidation(BaseStrategy): @property def ceilometer(self): if self._ceilometer is None: - self._ceilometer = CeilometerClusterHistory() + self._ceilometer = CeilometerClusterHistory(osc=self.osc) return self._ceilometer @ceilometer.setter diff --git a/watcher/decision_engine/strategy/strategies/dummy_strategy.py b/watcher/decision_engine/strategy/strategies/dummy_strategy.py index a9b3340b1..6a6eeff67 100644 --- a/watcher/decision_engine/strategy/strategies/dummy_strategy.py +++ b/watcher/decision_engine/strategy/strategies/dummy_strategy.py @@ -30,8 +30,9 @@ class DummyStrategy(BaseStrategy): NOP = "nop" SLEEP = "sleep" - def __init__(self, name=DEFAULT_NAME, description=DEFAULT_DESCRIPTION): - super(DummyStrategy, self).__init__(name, description) + def __init__(self, name=DEFAULT_NAME, description=DEFAULT_DESCRIPTION, + osc=None): + super(DummyStrategy, self).__init__(name, description, osc) def execute(self, model): parameters = {'message': 'hello World'} diff --git a/watcher/decision_engine/strategy/strategies/outlet_temp_control.py b/watcher/decision_engine/strategy/strategies/outlet_temp_control.py index 7b7f3954b..4b8a86589 100644 --- a/watcher/decision_engine/strategy/strategies/outlet_temp_control.py +++ b/watcher/decision_engine/strategy/strategies/outlet_temp_control.py @@ -40,7 +40,8 @@ class OutletTempControl(BaseStrategy): MIGRATION = "migrate" - def __init__(self, name=DEFAULT_NAME, description=DEFAULT_DESCRIPTION): + def __init__(self, name=DEFAULT_NAME, description=DEFAULT_DESCRIPTION, + osc=None): """[PoC]Outlet temperature control using live migration It is a migration strategy based on the Outlet Temperature of physical @@ -67,8 +68,9 @@ class OutletTempControl(BaseStrategy): :param name: the name of the strategy :param description: a description of the strategy + :param osc: an OpenStackClients object """ - super(OutletTempControl, self).__init__(name, description) + super(OutletTempControl, self).__init__(name, description, osc) # the migration plan will be triggered when the outlet temperature # reaches threshold # TODO(zhenzanz): Threshold should be configurable for each audit @@ -79,7 +81,7 @@ class OutletTempControl(BaseStrategy): @property def ceilometer(self): if self._ceilometer is None: - self._ceilometer = CeilometerClusterHistory() + self._ceilometer = CeilometerClusterHistory(osc=self.osc) return self._ceilometer @ceilometer.setter diff --git a/watcher/metrics_engine/cluster_history/ceilometer.py b/watcher/metrics_engine/cluster_history/ceilometer.py index a1faee275..80f235869 100644 --- a/watcher/metrics_engine/cluster_history/ceilometer.py +++ b/watcher/metrics_engine/cluster_history/ceilometer.py @@ -20,7 +20,7 @@ from oslo_config import cfg from oslo_log import log -from watcher.common.ceilometer import CeilometerClient +from watcher.common import ceilometer_helper from watcher.metrics_engine.cluster_history.api import BaseClusterHistory @@ -29,8 +29,10 @@ LOG = log.getLogger(__name__) class CeilometerClusterHistory(BaseClusterHistory): - def __init__(self): - self.ceilometer = CeilometerClient() + def __init__(self, osc=None): + """:param osc: an OpenStackClients instance""" + super(CeilometerClusterHistory, self).__init__() + self.ceilometer = ceilometer_helper.CeilometerHelper(osc=osc) def statistic_list(self, meter_name, query=None, period=None): return self.ceilometer.statistic_list(meter_name, query, period) diff --git a/watcher/metrics_engine/cluster_model_collector/manager.py b/watcher/metrics_engine/cluster_model_collector/manager.py index e138ea8a0..49369264e 100644 --- a/watcher/metrics_engine/cluster_model_collector/manager.py +++ b/watcher/metrics_engine/cluster_model_collector/manager.py @@ -20,8 +20,7 @@ from oslo_config import cfg from oslo_log import log -from watcher.common.keystone import KeystoneClient -from watcher.common.nova import NovaClient +from watcher.common import nova_helper from watcher.metrics_engine.cluster_model_collector.nova import \ NovaClusterModelCollector @@ -30,8 +29,7 @@ CONF = cfg.CONF class CollectorManager(object): - def get_cluster_model_collector(self): - keystone = KeystoneClient() - wrapper = NovaClient(keystone.get_credentials(), - session=keystone.get_session()) - return NovaClusterModelCollector(wrapper=wrapper) + def get_cluster_model_collector(self, osc=None): + """:param osc: an OpenStackClients instance""" + nova = nova_helper.NovaHelper(osc=osc) + return NovaClusterModelCollector(nova) diff --git a/watcher/opts.py b/watcher/opts.py index d2d287047..af237d638 100644 --- a/watcher/opts.py +++ b/watcher/opts.py @@ -15,8 +15,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from keystoneauth1 import loading as ka_loading + import watcher.api.app from watcher.applier import manager as applier_manager +from watcher.common import clients from watcher.decision_engine import manager as decision_engine_manger from watcher.decision_engine.planner import manager as planner_manager from watcher.decision_engine.strategy.selection import default \ @@ -29,7 +32,15 @@ def list_opts(): ('watcher_goals', strategy_selector.WATCHER_GOALS_OPTS), ('watcher_decision_engine', decision_engine_manger.WATCHER_DECISION_ENGINE_OPTS), - ('watcher_applier', - applier_manager.APPLIER_MANAGER_OPTS), - ('watcher_planner', planner_manager.WATCHER_PLANNER_OPTS) + ('watcher_applier', applier_manager.APPLIER_MANAGER_OPTS), + ('watcher_planner', planner_manager.WATCHER_PLANNER_OPTS), + ('nova_client', clients.NOVA_CLIENT_OPTS), + ('glance_client', clients.GLANCE_CLIENT_OPTS), + ('cinder_client', clients.CINDER_CLIENT_OPTS), + ('ceilometer_client', clients.CEILOMETER_CLIENT_OPTS), + ('neutron_client', clients.NEUTRON_CLIENT_OPTS), + ('watcher_clients_auth', + (ka_loading.get_auth_common_conf_options() + + ka_loading.get_auth_plugin_conf_options('password') + + ka_loading.get_session_conf_options())) ] diff --git a/watcher/tests/applier/workflow_engine/test_default_workflow_engine.py b/watcher/tests/applier/workflow_engine/test_default_workflow_engine.py index 8a183095a..f48ea92f7 100644 --- a/watcher/tests/applier/workflow_engine/test_default_workflow_engine.py +++ b/watcher/tests/applier/workflow_engine/test_default_workflow_engine.py @@ -53,9 +53,9 @@ class FakeAction(abase.BaseAction): class TestDefaultWorkFlowEngine(base.DbTestCase): def setUp(self): super(TestDefaultWorkFlowEngine, self).setUp() - self.engine = tflow.DefaultWorkFlowEngine() - self.engine.context = self.context - self.engine.applier_manager = mock.MagicMock() + self.engine = tflow.DefaultWorkFlowEngine( + context=self.context, + applier_manager=mock.MagicMock()) def test_execute(self): actions = mock.MagicMock() diff --git a/watcher/tests/common/test_ceilometer.py b/watcher/tests/common/test_ceilometer.py deleted file mode 100644 index 78f2ebc71..000000000 --- a/watcher/tests/common/test_ceilometer.py +++ /dev/null @@ -1,99 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from __future__ import unicode_literals - -from mock import MagicMock -from mock import mock -from oslo_config import cfg -from watcher.common.ceilometer import CeilometerClient - -from watcher.tests.base import BaseTestCase -CONF = cfg.CONF - - -class TestCeilometer(BaseTestCase): - def setUp(self): - super(TestCeilometer, self).setUp() - self.cm = CeilometerClient() - - def test_build_query(self): - expected = [{'field': 'user_id', 'op': 'eq', 'value': u'user_id'}, - {'field': 'project_id', 'op': 'eq', 'value': u'tenant_id'}, - {'field': 'resource_id', 'op': 'eq', - 'value': u'resource_id'}] - - query = self.cm.build_query(user_id="user_id", - tenant_id="tenant_id", - resource_id="resource_id", - user_ids=["user_ids"], - tenant_ids=["tenant_ids"], - resource_ids=["resource_ids"]) - self.assertEqual(query, expected) - - @mock.patch('keystoneclient.v2_0.client.Client', autospec=True) - @mock.patch('ceilometerclient.v2.client.Client', autospec=True) - def test_get_ceilometer_v2(self, mock_keystone, mock_ceilometer): - cfg.CONF.set_override( - 'auth_uri', "http://127.0.0.1:9898/v2", group="keystone_authtoken", - enforce_type=True - ) - c = CeilometerClient(api_version='2') - from ceilometerclient.v2 import Client - self.assertIsInstance(c.cmclient, Client) - - @mock.patch.object(CeilometerClient, "cmclient") - def test_statistic_aggregation(self, mock_keystone): - statistic = MagicMock() - expected_result = 100 - statistic[-1]._info = {'aggregate': {'avg': expected_result}} - mock_keystone.statistics.list.return_value = statistic - val = self.cm.statistic_aggregation( - resource_id="VM_ID", - meter_name="cpu_util", - period="7300" - ) - self.assertEqual(val, expected_result) - - @mock.patch.object(CeilometerClient, "cmclient") - def test_get_last_sample(self, mock_keystone): - statistic = MagicMock() - expected_result = 100 - statistic[-1]._info = {'counter_volume': expected_result} - mock_keystone.samples.list.return_value = statistic - val = self.cm.get_last_sample_value( - resource_id="id", - meter_name="compute.node.percent" - ) - self.assertEqual(val, expected_result) - - @mock.patch.object(CeilometerClient, "cmclient") - def test_get_last_sample_none(self, mock_keystone): - expected = [] - mock_keystone.samples.list.return_value = expected - val = self.cm.get_last_sample_values( - resource_id="id", - meter_name="compute.node.percent" - ) - self.assertEqual(val, expected) - - @mock.patch.object(CeilometerClient, "cmclient") - def test_statistic_list(self, mock_keystone): - expected_value = [] - mock_keystone.statistics.list.return_value = expected_value - val = self.cm.statistic_list(meter_name="cpu_util") - self.assertEqual(val, expected_value) diff --git a/watcher/tests/common/test_ceilometer_helper.py b/watcher/tests/common/test_ceilometer_helper.py new file mode 100644 index 000000000..3430c2c6f --- /dev/null +++ b/watcher/tests/common/test_ceilometer_helper.py @@ -0,0 +1,98 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import unicode_literals + +import mock +from oslo_config import cfg + +from watcher.common import ceilometer_helper +from watcher.common import clients +from watcher.tests import base + +CONF = cfg.CONF + + +@mock.patch.object(clients.OpenStackClients, 'ceilometer') +class TestCeilometerHelper(base.BaseTestCase): + + def test_build_query(self, mock_ceilometer): + mock_ceilometer.return_value = mock.MagicMock() + cm = ceilometer_helper.CeilometerHelper() + expected = [{'field': 'user_id', 'op': 'eq', 'value': u'user_id'}, + {'field': 'project_id', 'op': 'eq', 'value': u'tenant_id'}, + {'field': 'resource_id', 'op': 'eq', + 'value': u'resource_id'}] + + query = cm.build_query(user_id="user_id", + tenant_id="tenant_id", + resource_id="resource_id", + user_ids=["user_ids"], + tenant_ids=["tenant_ids"], + resource_ids=["resource_ids"]) + self.assertEqual(query, expected) + + def test_statistic_aggregation(self, mock_ceilometer): + cm = ceilometer_helper.CeilometerHelper() + ceilometer = mock.MagicMock() + statistic = mock.MagicMock() + expected_result = 100 + statistic[-1]._info = {'aggregate': {'avg': expected_result}} + ceilometer.statistics.list.return_value = statistic + mock_ceilometer.return_value = ceilometer + cm = ceilometer_helper.CeilometerHelper() + val = cm.statistic_aggregation( + resource_id="VM_ID", + meter_name="cpu_util", + period="7300" + ) + self.assertEqual(val, expected_result) + + def test_get_last_sample(self, mock_ceilometer): + ceilometer = mock.MagicMock() + statistic = mock.MagicMock() + expected_result = 100 + statistic[-1]._info = {'counter_volume': expected_result} + ceilometer.samples.list.return_value = statistic + mock_ceilometer.return_value = ceilometer + cm = ceilometer_helper.CeilometerHelper() + val = cm.get_last_sample_value( + resource_id="id", + meter_name="compute.node.percent" + ) + self.assertEqual(val, expected_result) + + def test_get_last_sample_none(self, mock_ceilometer): + ceilometer = mock.MagicMock() + expected = [] + ceilometer.samples.list.return_value = expected + mock_ceilometer.return_value = ceilometer + cm = ceilometer_helper.CeilometerHelper() + val = cm.get_last_sample_values( + resource_id="id", + meter_name="compute.node.percent" + ) + self.assertEqual(val, expected) + + def test_statistic_list(self, mock_ceilometer): + ceilometer = mock.MagicMock() + expected_value = [] + ceilometer.statistics.list.return_value = expected_value + mock_ceilometer.return_value = ceilometer + cm = ceilometer_helper.CeilometerHelper() + val = cm.statistic_list(meter_name="cpu_util") + self.assertEqual(val, expected_value) diff --git a/watcher/tests/common/test_clients.py b/watcher/tests/common/test_clients.py new file mode 100644 index 000000000..82a394c3d --- /dev/null +++ b/watcher/tests/common/test_clients.py @@ -0,0 +1,241 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from ceilometerclient import client as ceclient +import ceilometerclient.v2.client as ceclient_v2 +from cinderclient import client as ciclient +from cinderclient.v1 import client as ciclient_v1 +from glanceclient import client as glclient +from keystoneauth1 import loading as ka_loading +import mock +from neutronclient.neutron import client as netclient +from neutronclient.v2_0 import client as netclient_v2 +from novaclient import client as nvclient +from oslo_config import cfg + +from watcher.common import clients +from watcher.tests import base + + +class TestClients(base.BaseTestCase): + + def setUp(self): + super(TestClients, self).setUp() + + cfg.CONF.import_opt('api_version', 'watcher.common.clients', + group='nova_client') + cfg.CONF.import_opt('api_version', 'watcher.common.clients', + group='glance_client') + cfg.CONF.import_opt('api_version', 'watcher.common.clients', + group='cinder_client') + cfg.CONF.import_opt('api_version', 'watcher.common.clients', + group='ceilometer_client') + cfg.CONF.import_opt('api_version', 'watcher.common.clients', + group='neutron_client') + + def test_get_keystone_session(self): + _AUTH_CONF_GROUP = 'watcher_clients_auth' + ka_loading.register_auth_conf_options(cfg.CONF, _AUTH_CONF_GROUP) + ka_loading.register_session_conf_options(cfg.CONF, _AUTH_CONF_GROUP) + + cfg.CONF.set_override('auth_type', 'password', + group=_AUTH_CONF_GROUP) + + # If we don't clean up the _AUTH_CONF_GROUP conf options, then other + # tests that run after this one will fail, complaining about required + # options that _AUTH_CONF_GROUP wants. + def cleanup_conf_from_loading(): + # oslo_config doesn't seem to allow unregistering groups through a + # single method, so we do this instead + cfg.CONF.reset() + del cfg.CONF._groups[_AUTH_CONF_GROUP] + + self.addCleanup(cleanup_conf_from_loading) + + osc = clients.OpenStackClients() + + expected = {'username': 'foousername', + 'password': 'foopassword', + 'auth_url': 'http://server.ip:35357', + 'user_domain_id': 'foouserdomainid', + 'project_domain_id': 'fooprojdomainid'} + + def reset_register_opts_mock(conf_obj, original_method): + conf_obj.register_opts = original_method + + original_register_opts = cfg.CONF.register_opts + self.addCleanup(reset_register_opts_mock, + cfg.CONF, + original_register_opts) + + # Because some of the conf options for auth plugins are not registered + # until right before they are loaded, and because the method that does + # the actual loading of the conf option values is an anonymous method + # (see _getter method of load_from_conf_options in + # keystoneauth1.loading.conf.py), we need to manually monkey patch + # the register opts method so that we can override the conf values to + # our custom values. + def mock_register_opts(*args, **kwargs): + ret = original_register_opts(*args, **kwargs) + if 'group' in kwargs and kwargs['group'] == _AUTH_CONF_GROUP: + for key, value in expected.items(): + cfg.CONF.set_override(key, value, group=_AUTH_CONF_GROUP) + return ret + + cfg.CONF.register_opts = mock_register_opts + + sess = osc.session + self.assertEqual(expected['auth_url'], sess.auth.auth_url) + self.assertEqual(expected['username'], sess.auth._username) + self.assertEqual(expected['password'], sess.auth._password) + self.assertEqual(expected['user_domain_id'], sess.auth._user_domain_id) + self.assertEqual(expected['project_domain_id'], + sess.auth._project_domain_id) + + @mock.patch.object(nvclient, 'Client') + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_nova(self, mock_session, mock_call): + osc = clients.OpenStackClients() + osc._nova = None + osc.nova() + mock_call.assert_called_once_with(cfg.CONF.nova_client.api_version, + session=mock_session) + + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_nova_diff_vers(self, mock_session): + cfg.CONF.set_override('api_version', '2.3', + group='nova_client') + osc = clients.OpenStackClients() + osc._nova = None + osc.nova() + self.assertEqual('2.3', osc.nova().api_version.get_string()) + + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_nova_cached(self, mock_session): + osc = clients.OpenStackClients() + osc._nova = None + nova = osc.nova() + nova_cached = osc.nova() + self.assertEqual(nova, nova_cached) + + @mock.patch.object(glclient, 'Client') + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_glance(self, mock_session, mock_call): + osc = clients.OpenStackClients() + osc._glance = None + osc.glance() + mock_call.assert_called_once_with(cfg.CONF.glance_client.api_version, + session=mock_session) + + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_glance_diff_vers(self, mock_session): + cfg.CONF.set_override('api_version', '1', + group='glance_client') + osc = clients.OpenStackClients() + osc._glance = None + osc.glance() + self.assertEqual(1.0, osc.glance().version) + + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_glance_cached(self, mock_session): + osc = clients.OpenStackClients() + osc._glance = None + glance = osc.glance() + glance_cached = osc.glance() + self.assertEqual(glance, glance_cached) + + @mock.patch.object(ciclient, 'Client') + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_cinder(self, mock_session, mock_call): + osc = clients.OpenStackClients() + osc._cinder = None + osc.cinder() + mock_call.assert_called_once_with(cfg.CONF.cinder_client.api_version, + session=mock_session) + + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_cinder_diff_vers(self, mock_session): + cfg.CONF.set_override('api_version', '1', + group='cinder_client') + osc = clients.OpenStackClients() + osc._cinder = None + osc.cinder() + self.assertEqual(ciclient_v1.Client, type(osc.cinder())) + + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_cinder_cached(self, mock_session): + osc = clients.OpenStackClients() + osc._cinder = None + cinder = osc.cinder() + cinder_cached = osc.cinder() + self.assertEqual(cinder, cinder_cached) + + @mock.patch.object(ceclient, 'Client') + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_ceilometer(self, mock_session, mock_call): + osc = clients.OpenStackClients() + osc._ceilometer = None + osc.ceilometer() + mock_call.assert_called_once_with( + cfg.CONF.ceilometer_client.api_version, + session=mock_session) + + @mock.patch.object(clients.OpenStackClients, 'session') + @mock.patch.object(ceclient_v2.Client, '_get_alarm_client') + def test_clients_ceilometer_diff_vers(self, mock_get_alarm_client, + mock_session): + '''ceilometerclient currently only has one version (v2)''' + cfg.CONF.set_override('api_version', '2', + group='ceilometer_client') + osc = clients.OpenStackClients() + osc._ceilometer = None + osc.ceilometer() + self.assertEqual(ceclient_v2.Client, + type(osc.ceilometer())) + + @mock.patch.object(clients.OpenStackClients, 'session') + @mock.patch.object(ceclient_v2.Client, '_get_alarm_client') + def test_clients_ceilometer_cached(self, mock_get_alarm_client, + mock_session): + osc = clients.OpenStackClients() + osc._ceilometer = None + ceilometer = osc.ceilometer() + ceilometer_cached = osc.ceilometer() + self.assertEqual(ceilometer, ceilometer_cached) + + @mock.patch.object(netclient, 'Client') + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_neutron(self, mock_session, mock_call): + osc = clients.OpenStackClients() + osc._neutron = None + osc.neutron() + mock_call.assert_called_once_with(cfg.CONF.neutron_client.api_version, + session=mock_session) + + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_neutron_diff_vers(self, mock_session): + '''neutronclient currently only has one version (v2)''' + cfg.CONF.set_override('api_version', '2', + group='neutron_client') + osc = clients.OpenStackClients() + osc._neutron = None + osc.neutron() + self.assertEqual(netclient_v2.Client, + type(osc.neutron())) + + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_neutron_cached(self, mock_session): + osc = clients.OpenStackClients() + osc._neutron = None + neutron = osc.neutron() + neutron_cached = osc.neutron() + self.assertEqual(neutron, neutron_cached) diff --git a/watcher/tests/common/test_keystone.py b/watcher/tests/common/test_keystone.py deleted file mode 100644 index bd284f2f2..000000000 --- a/watcher/tests/common/test_keystone.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from __future__ import unicode_literals -from keystoneclient.auth.identity import Password -from keystoneclient.session import Session -from mock import mock -from oslo_config import cfg -from watcher.common.keystone import KeystoneClient -from watcher.tests.base import BaseTestCase - -CONF = cfg.CONF - - -class TestKeystone(BaseTestCase): - def setUp(self): - super(TestKeystone, self).setUp() - self.ckeystone = KeystoneClient() - - @mock.patch('keystoneclient.v2_0.client.Client', autospec=True) - def test_get_endpoint_v2(self, keystone): - expected_endpoint = "http://ip:port/v2" - cfg.CONF.set_override( - 'auth_uri', expected_endpoint, group="keystone_authtoken", - enforce_type=True - ) - ks = mock.Mock() - ks.service_catalog.url_for.return_value = expected_endpoint - keystone.return_value = ks - ep = self.ckeystone.get_endpoint(service_type='metering', - endpoint_type='publicURL', - region_name='RegionOne') - - self.assertEqual(ep, expected_endpoint) - - @mock.patch('watcher.common.keystone.KeystoneClient._is_apiv3') - def test_get_session(self, mock_apiv3): - mock_apiv3.return_value = True - k = KeystoneClient() - session = k.get_session() - self.assertIsInstance(session.auth, Password) - self.assertIsInstance(session, Session) - - @mock.patch('watcher.common.keystone.KeystoneClient._is_apiv3') - def test_get_credentials(self, mock_apiv3): - mock_apiv3.return_value = True - expected_creds = {'auth_url': None, - 'password': None, - 'project_domain_name': 'default', - 'project_name': 'admin', - 'user_domain_name': 'default', - 'username': None} - creds = self.ckeystone.get_credentials() - self.assertEqual(creds, expected_creds) diff --git a/watcher/tests/common/test_nova_client.py b/watcher/tests/common/test_nova_client.py deleted file mode 100644 index fe22263a5..000000000 --- a/watcher/tests/common/test_nova_client.py +++ /dev/null @@ -1,153 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Authors: Jean-Emile DARTOIS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time - -import glanceclient.v2.client as glclient -import mock -import novaclient.client as nvclient - -from watcher.common import keystone -from watcher.common.nova import NovaClient -from watcher.common import utils -from watcher.tests import base - - -class TestNovaClient(base.TestCase): - - def setUp(self): - super(TestNovaClient, self).setUp() - self.instance_uuid = "fb5311b7-37f3-457e-9cde-6494a3c59bfe" - self.source_hypervisor = "ldev-indeedsrv005" - self.destination_hypervisor = "ldev-indeedsrv006" - - self.creds = mock.MagicMock() - self.session = mock.MagicMock() - - @mock.patch.object(keystone, 'KeystoneClient', mock.Mock()) - @mock.patch.object(nvclient, "Client", mock.Mock()) - def test_stop_instance(self): - nova_client = NovaClient(creds=self.creds, session=self.session) - instance_id = utils.generate_uuid() - server = mock.MagicMock() - server.id = instance_id - setattr(server, 'OS-EXT-STS:vm_state', 'stopped') - nova_client.nova.servers = mock.MagicMock() - nova_client.nova.servers.find.return_value = server - nova_client.nova.servers.list.return_value = [server] - - result = nova_client.stop_instance(instance_id) - self.assertEqual(result, True) - - @mock.patch.object(keystone, 'KeystoneClient', mock.Mock()) - @mock.patch.object(nvclient, "Client", mock.Mock()) - def test_set_host_offline(self): - nova_client = NovaClient(creds=self.creds, session=self.session) - host = mock.MagicMock() - nova_client.nova.hosts = mock.MagicMock() - nova_client.nova.hosts.get.return_value = host - result = nova_client.set_host_offline("rennes") - self.assertEqual(result, True) - - @mock.patch.object(time, 'sleep', mock.Mock()) - @mock.patch.object(keystone, 'KeystoneClient', mock.Mock()) - @mock.patch.object(nvclient, "Client", mock.Mock()) - def test_live_migrate_instance(self): - nova_client = NovaClient(creds=self.creds, session=self.session) - server = mock.MagicMock() - server.id = self.instance_uuid - nova_client.nova.servers = mock.MagicMock() - nova_client.nova.servers.list.return_value = [server] - instance = nova_client.live_migrate_instance( - self.instance_uuid, self.destination_hypervisor - ) - self.assertIsNotNone(instance) - - @mock.patch.object(keystone, 'KeystoneClient', mock.Mock()) - @mock.patch.object(nvclient, "Client", mock.Mock()) - def test_watcher_non_live_migrate_instance_not_found(self): - nova_client = NovaClient(creds=self.creds, session=self.session) - nova_client.nova.servers.list.return_value = [] - nova_client.nova.servers.find.return_value = None - - is_success = nova_client.watcher_non_live_migrate_instance( - self.instance_uuid, - self.destination_hypervisor) - - self.assertEqual(is_success, False) - - @mock.patch.object(time, 'sleep', mock.Mock()) - @mock.patch.object(keystone, 'KeystoneClient', mock.Mock()) - @mock.patch.object(nvclient, "Client", mock.Mock()) - def test_watcher_non_live_migrate_instance_volume(self): - nova_client = NovaClient(creds=self.creds, session=self.session) - instance = mock.MagicMock(id=self.instance_uuid) - setattr(instance, 'OS-EXT-SRV-ATTR:host', self.source_hypervisor) - nova_client.nova.servers.list.return_value = [instance] - nova_client.nova.servers.find.return_value = instance - instance = nova_client.watcher_non_live_migrate_instance( - self.instance_uuid, - self.destination_hypervisor) - self.assertIsNotNone(instance) - - @mock.patch.object(time, 'sleep', mock.Mock()) - @mock.patch.object(keystone, 'KeystoneClient', mock.Mock()) - @mock.patch.object(nvclient, "Client", mock.Mock()) - def test_watcher_non_live_migrate_keep_image(self): - nova_client = NovaClient(creds=self.creds, session=self.session) - instance = mock.MagicMock(id=self.instance_uuid) - setattr(instance, 'OS-EXT-SRV-ATTR:host', self.source_hypervisor) - addresses = mock.MagicMock() - type = mock.MagicMock() - networks = [] - networks.append(("lan", type)) - addresses.items.return_value = networks - attached_volumes = mock.MagicMock() - setattr(instance, 'addresses', addresses) - setattr(instance, "os-extended-volumes:volumes_attached", - attached_volumes) - nova_client.nova.servers.list.return_value = [instance] - nova_client.nova.servers.find.return_value = instance - instance = nova_client.watcher_non_live_migrate_instance( - self.instance_uuid, - self.destination_hypervisor, keep_original_image_name=False) - self.assertIsNotNone(instance) - - @mock.patch.object(time, 'sleep', mock.Mock()) - @mock.patch.object(keystone, 'KeystoneClient', mock.Mock()) - @mock.patch.object(nvclient, "Client", mock.Mock()) - @mock.patch.object(glclient, "Client") - def test_create_image_from_instance(self, m_glance_cls): - nova_client = NovaClient(creds=self.creds, session=self.session) - instance = mock.MagicMock() - image = mock.MagicMock() - setattr(instance, 'OS-EXT-SRV-ATTR:host', self.source_hypervisor) - nova_client.nova.servers.list.return_value = [instance] - nova_client.nova.servers.find.return_value = instance - image_uuid = 'fake-image-uuid' - nova_client.nova.servers.create_image.return_value = image - - m_glance = mock.MagicMock() - m_glance_cls.return_value = m_glance - - m_glance.images = {image_uuid: image} - instance = nova_client.create_image_from_instance( - self.instance_uuid, "Cirros" - ) - self.assertIsNone(instance) diff --git a/watcher/tests/common/test_nova_helper.py b/watcher/tests/common/test_nova_helper.py new file mode 100644 index 000000000..f705e6b2f --- /dev/null +++ b/watcher/tests/common/test_nova_helper.py @@ -0,0 +1,144 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time + +import mock + +from watcher.common import clients +from watcher.common import nova_helper +from watcher.common import utils +from watcher.tests import base + + +@mock.patch.object(clients.OpenStackClients, 'nova') +@mock.patch.object(clients.OpenStackClients, 'neutron') +@mock.patch.object(clients.OpenStackClients, 'cinder') +@mock.patch.object(clients.OpenStackClients, 'glance') +class TestNovaHelper(base.TestCase): + + def setUp(self): + super(TestNovaHelper, self).setUp() + self.instance_uuid = "fb5311b7-37f3-457e-9cde-6494a3c59bfe" + self.source_hypervisor = "ldev-indeedsrv005" + self.destination_hypervisor = "ldev-indeedsrv006" + + def test_stop_instance(self, mock_glance, mock_cinder, mock_neutron, + mock_nova): + nova_util = nova_helper.NovaHelper() + instance_id = utils.generate_uuid() + server = mock.MagicMock() + server.id = instance_id + setattr(server, 'OS-EXT-STS:vm_state', 'stopped') + nova_util.nova.servers = mock.MagicMock() + nova_util.nova.servers.find.return_value = server + nova_util.nova.servers.list.return_value = [server] + + result = nova_util.stop_instance(instance_id) + self.assertEqual(result, True) + + def test_set_host_offline(self, mock_glance, mock_cinder, mock_neutron, + mock_nova): + nova_util = nova_helper.NovaHelper() + host = mock.MagicMock() + nova_util.nova.hosts = mock.MagicMock() + nova_util.nova.hosts.get.return_value = host + result = nova_util.set_host_offline("rennes") + self.assertEqual(result, True) + + @mock.patch.object(time, 'sleep', mock.Mock()) + def test_live_migrate_instance(self, mock_glance, mock_cinder, + mock_neutron, mock_nova): + nova_util = nova_helper.NovaHelper() + server = mock.MagicMock() + server.id = self.instance_uuid + nova_util.nova.servers = mock.MagicMock() + nova_util.nova.servers.list.return_value = [server] + instance = nova_util.live_migrate_instance( + self.instance_uuid, self.destination_hypervisor + ) + self.assertIsNotNone(instance) + + def test_watcher_non_live_migrate_instance_not_found( + self, mock_glance, mock_cinder, mock_neutron, mock_nova): + nova_util = nova_helper.NovaHelper() + nova_util.nova.servers.list.return_value = [] + nova_util.nova.servers.find.return_value = None + + is_success = nova_util.watcher_non_live_migrate_instance( + self.instance_uuid, + self.destination_hypervisor) + + self.assertEqual(is_success, False) + + @mock.patch.object(time, 'sleep', mock.Mock()) + def test_watcher_non_live_migrate_instance_volume( + self, mock_glance, mock_cinder, mock_neutron, mock_nova): + nova_util = nova_helper.NovaHelper() + instance = mock.MagicMock(id=self.instance_uuid) + setattr(instance, 'OS-EXT-SRV-ATTR:host', self.source_hypervisor) + nova_util.nova.servers.list.return_value = [instance] + nova_util.nova.servers.find.return_value = instance + instance = nova_util.watcher_non_live_migrate_instance( + self.instance_uuid, + self.destination_hypervisor) + self.assertIsNotNone(instance) + + @mock.patch.object(time, 'sleep', mock.Mock()) + def test_watcher_non_live_migrate_keep_image( + self, mock_glance, mock_cinder, mock_neutron, mock_nova): + nova_util = nova_helper.NovaHelper() + instance = mock.MagicMock(id=self.instance_uuid) + setattr(instance, 'OS-EXT-SRV-ATTR:host', self.source_hypervisor) + addresses = mock.MagicMock() + type = mock.MagicMock() + networks = [] + networks.append(("lan", type)) + addresses.items.return_value = networks + attached_volumes = mock.MagicMock() + setattr(instance, 'addresses', addresses) + setattr(instance, "os-extended-volumes:volumes_attached", + attached_volumes) + nova_util.nova.servers.list.return_value = [instance] + nova_util.nova.servers.find.return_value = instance + instance = nova_util.watcher_non_live_migrate_instance( + self.instance_uuid, + self.destination_hypervisor, keep_original_image_name=False) + self.assertIsNotNone(instance) + + @mock.patch.object(time, 'sleep', mock.Mock()) + def test_create_image_from_instance(self, mock_glance, mock_cinder, + mock_neutron, mock_nova): + nova_util = nova_helper.NovaHelper() + instance = mock.MagicMock() + image = mock.MagicMock() + setattr(instance, 'OS-EXT-SRV-ATTR:host', self.source_hypervisor) + nova_util.nova.servers.list.return_value = [instance] + nova_util.nova.servers.find.return_value = instance + image_uuid = 'fake-image-uuid' + nova_util.nova.servers.create_image.return_value = image + + glance_client = mock.MagicMock() + mock_glance.return_value = glance_client + + glance_client.images = {image_uuid: image} + instance = nova_util.create_image_from_instance( + self.instance_uuid, "Cirros" + ) + self.assertIsNone(instance) diff --git a/watcher/tests/decision_engine/strategy/selector/test_strategy_selector.py b/watcher/tests/decision_engine/strategy/selector/test_strategy_selector.py index 144ba53f4..516bbbec5 100644 --- a/watcher/tests/decision_engine/strategy/selector/test_strategy_selector.py +++ b/watcher/tests/decision_engine/strategy/selector/test_strategy_selector.py @@ -36,8 +36,8 @@ class TestStrategySelector(TestCase): enforce_type=True) expected_goal = 'DUMMY' expected_strategy = CONF.watcher_goals.goals[expected_goal] - self.strategy_selector.define_from_goal(expected_goal) - mock_call.assert_called_once_with(expected_strategy) + self.strategy_selector.define_from_goal(expected_goal, osc=None) + mock_call.assert_called_once_with(expected_strategy, osc=None) @patch.object(DefaultStrategyLoader, 'load') def test_define_from_goal_with_incorrect_mapping(self, mock_call):