Database tunability

Add additional tunables to how SQLalchemy creates connection pools

Also start including static doc assets to they can be published

Change-Id: I268dc265a6b6cf1a200b235a5f99e65e89a95637

.gitignore

@@ -63,7 +63,6 @@ instance/
 
 # Sphinx documentation
 docs/_build/
-docs/*/_static/
 
 # PyBuilder
 target/

docs/source/_static/drydock.conf.sample

@@ -0,0 +1,402 @@
+[DEFAULT]
+
+#
+# From drydock_provisioner
+#
+
+# Polling interval in seconds for checking subtask or downstream status (integer
+# value)
+#poll_interval = 10
+
+# How long a leader has to check-in before leadership can be usurped, in seconds
+# (integer value)
+#leader_grace_period = 300
+
+# How often will an instance attempt to claim leadership, in seconds (integer
+# value)
+#leadership_claim_interval = 30
+
+
+[database]
+
+#
+# From drydock_provisioner
+#
+
+# The URI database connect string. (string value)
+#database_connect_string = <None>
+
+# The SQLalchemy database connection pool size. (integer value)
+#pool_size = 15
+
+# Should DB connections be validated prior to use. (boolean value)
+#pool_pre_ping = true
+
+# How long a request for a connection should wait before one becomes available.
+# (integer value)
+#pool_timeout = 30
+
+# How many connections above pool_size are allowed to be open during high usage.
+# (integer value)
+#pool_overflow = 10
+
+# Time, in seconds, when a connection should be closed and re-established. -1
+# for no recycling. (integer value)
+#connection_recycle = -1
+
+
+[keystone_authtoken]
+
+#
+# From drydock_provisioner
+#
+
+# Authentication URL (string value)
+#auth_url = <None>
+
+# Domain ID to scope to (string value)
+#domain_id = <None>
+
+# Domain name to scope to (string value)
+#domain_name = <None>
+
+# Project ID to scope to (string value)
+# Deprecated group/name - [keystone_authtoken]/tenant_id
+#project_id = <None>
+
+# Project name to scope to (string value)
+# Deprecated group/name - [keystone_authtoken]/tenant_name
+#project_name = <None>
+
+# Domain ID containing project (string value)
+#project_domain_id = <None>
+
+# Domain name containing project (string value)
+#project_domain_name = <None>
+
+# Trust ID (string value)
+#trust_id = <None>
+
+# Optional domain ID to use with v3 and v2 parameters. It will be used for both
+# the user and project domain in v3 and ignored in v2 authentication. (string
+# value)
+#default_domain_id = <None>
+
+# Optional domain name to use with v3 API and v2 parameters. It will be used for
+# both the user and project domain in v3 and ignored in v2 authentication.
+# (string value)
+#default_domain_name = <None>
+
+# User id (string value)
+#user_id = <None>
+
+# Username (string value)
+# Deprecated group/name - [keystone_authtoken]/user_name
+#username = <None>
+
+# User's domain id (string value)
+#user_domain_id = <None>
+
+# User's domain name (string value)
+#user_domain_name = <None>
+
+# User's password (string value)
+#password = <None>
+
+#
+# From keystonemiddleware.auth_token
+#
+
+# Complete "public" Identity API endpoint. This endpoint should not be an
+# "admin" endpoint, as it should be accessible by all end users. Unauthenticated
+# clients are redirected to this endpoint to authenticate. Although this
+# endpoint should  ideally be unversioned, client support in the wild varies.
+# If you're using a versioned v2 endpoint here, then this  should *not* be the
+# same endpoint the service user utilizes  for validating tokens, because normal
+# end users may not be  able to reach that endpoint. (string value)
+#auth_uri = <None>
+
+# API version of the admin Identity API endpoint. (string value)
+#auth_version = <None>
+
+# Do not handle authorization requests within the middleware, but delegate the
+# authorization decision to downstream WSGI components. (boolean value)
+#delay_auth_decision = false
+
+# Request timeout value for communicating with Identity API server. (integer
+# value)
+#http_connect_timeout = <None>
+
+# How many times are we trying to reconnect when communicating with Identity API
+# Server. (integer value)
+#http_request_max_retries = 3
+
+# Request environment key where the Swift cache object is stored. When
+# auth_token middleware is deployed with a Swift cache, use this option to have
+# the middleware share a caching backend with swift. Otherwise, use the
+# ``memcached_servers`` option instead. (string value)
+#cache = <None>
+
+# Required if identity server requires client certificate (string value)
+#certfile = <None>
+
+# Required if identity server requires client certificate (string value)
+#keyfile = <None>
+
+# A PEM encoded Certificate Authority to use when verifying HTTPs connections.
+# Defaults to system CAs. (string value)
+#cafile = <None>
+
+# Verify HTTPS connections. (boolean value)
+#insecure = false
+
+# The region in which the identity server can be found. (string value)
+#region_name = <None>
+
+# Directory used to cache files related to PKI tokens. (string value)
+#signing_dir = <None>
+
+# Optionally specify a list of memcached server(s) to use for caching. If left
+# undefined, tokens will instead be cached in-process. (list value)
+# Deprecated group/name - [keystone_authtoken]/memcache_servers
+#memcached_servers = <None>
+
+# In order to prevent excessive effort spent validating tokens, the middleware
+# caches previously-seen tokens for a configurable duration (in seconds). Set to
+# -1 to disable caching completely. (integer value)
+#token_cache_time = 300
+
+# Determines the frequency at which the list of revoked tokens is retrieved from
+# the Identity service (in seconds). A high number of revocation events combined
+# with a low cache duration may significantly reduce performance. Only valid for
+# PKI tokens. (integer value)
+#revocation_cache_time = 10
+
+# (Optional) If defined, indicate whether token data should be authenticated or
+# authenticated and encrypted. If MAC, token data is authenticated (with HMAC)
+# in the cache. If ENCRYPT, token data is encrypted and authenticated in the
+# cache. If the value is not one of these options or empty, auth_token will
+# raise an exception on initialization. (string value)
+# Possible values:
+# None - <No description provided>
+# MAC - <No description provided>
+# ENCRYPT - <No description provided>
+#memcache_security_strategy = None
+
+# (Optional, mandatory if memcache_security_strategy is defined) This string is
+# used for key derivation. (string value)
+#memcache_secret_key = <None>
+
+# (Optional) Number of seconds memcached server is considered dead before it is
+# tried again. (integer value)
+#memcache_pool_dead_retry = 300
+
+# (Optional) Maximum total number of open connections to every memcached server.
+# (integer value)
+#memcache_pool_maxsize = 10
+
+# (Optional) Socket timeout in seconds for communicating with a memcached
+# server. (integer value)
+#memcache_pool_socket_timeout = 3
+
+# (Optional) Number of seconds a connection to memcached is held unused in the
+# pool before it is closed. (integer value)
+#memcache_pool_unused_timeout = 60
+
+# (Optional) Number of seconds that an operation will wait to get a memcached
+# client connection from the pool. (integer value)
+#memcache_pool_conn_get_timeout = 10
+
+# (Optional) Use the advanced (eventlet safe) memcached client pool. The
+# advanced pool will only work under python 2.x. (boolean value)
+#memcache_use_advanced_pool = false
+
+# (Optional) Indicate whether to set the X-Service-Catalog header. If False,
+# middleware will not ask for service catalog on token validation and will not
+# set the X-Service-Catalog header. (boolean value)
+#include_service_catalog = true
+
+# Used to control the use and type of token binding. Can be set to: "disabled"
+# to not check token binding. "permissive" (default) to validate binding
+# information if the bind type is of a form known to the server and ignore it if
+# not. "strict" like "permissive" but if the bind type is unknown the token will
+# be rejected. "required" any form of token binding is needed to be allowed.
+# Finally the name of a binding method that must be present in tokens. (string
+# value)
+#enforce_token_bind = permissive
+
+# If true, the revocation list will be checked for cached tokens. This requires
+# that PKI tokens are configured on the identity server. (boolean value)
+#check_revocations_for_cached = false
+
+# Hash algorithms to use for hashing PKI tokens. This may be a single algorithm
+# or multiple. The algorithms are those supported by Python standard
+# hashlib.new(). The hashes will be tried in the order given, so put the
+# preferred one first for performance. The result of the first hash will be
+# stored in the cache. This will typically be set to multiple values only while
+# migrating from a less secure algorithm to a more secure one. Once all the old
+# tokens are expired this option should be set to a single value for better
+# performance. (list value)
+#hash_algorithms = md5
+
+# Authentication type to load (string value)
+# Deprecated group/name - [keystone_authtoken]/auth_plugin
+#auth_type = <None>
+
+# Config Section from which to load plugin specific options (string value)
+#auth_section = <None>
+
+
+[libvirt_driver]
+
+#
+# From drydock_provisioner
+#
+
+# Polling interval in seconds for querying libvirt status (integer value)
+#poll_interval = 10
+
+
+[logging]
+
+#
+# From drydock_provisioner
+#
+
+# Global log level for Drydock (string value)
+#log_level = INFO
+
+# Logger name for the top-level logger (string value)
+#global_logger_name = drydock_provisioner
+
+# Logger name for OOB driver logging (string value)
+#oobdriver_logger_name = ${global_logger_name}.oobdriver
+
+# Logger name for Node driver logging (string value)
+#nodedriver_logger_name = ${global_logger_name}.nodedriver
+
+# Logger name for API server logging (string value)
+#control_logger_name = ${global_logger_name}.control
+
+
+[maasdriver]
+
+#
+# From drydock_provisioner
+#
+
+# The API key for accessing MaaS (string value)
+#maas_api_key = <None>
+
+# The URL for accessing MaaS API (string value)
+#maas_api_url = <None>
+
+# Polling interval for querying MaaS status in seconds (integer value)
+#poll_interval = 10
+
+
+[network]
+
+#
+# From drydock_provisioner
+#
+
+# Timeout for initial read of outgoing HTTP calls from Drydock in seconds.
+# (integer value)
+#http_client_connect_timeout = 16
+
+# Timeout for initial read of outgoing HTTP calls from Drydock in seconds.
+# (integer value)
+#http_client_read_timeout = 300
+
+# Number of retries for transient errors of outgoing HTTP calls from Drydock.
+# (integer value)
+#http_client_retries = 3
+
+
+[oslo_policy]
+
+#
+# From oslo.policy
+#
+
+# The file that defines policies. (string value)
+#policy_file = policy.json
+
+# Default rule. Enforced when a requested rule is not found. (string value)
+#policy_default_rule = default
+
+# Directories where policy configuration files are stored. They can be relative
+# to any directory in the search path defined by the config_dir option, or
+# absolute paths. The file defined by policy_file must exist for these
+# directories to be searched.  Missing or empty directories are ignored. (multi
+# valued)
+#policy_dirs = policy.d
+
+
+[plugins]
+
+#
+# From drydock_provisioner
+#
+
+# Module path string of a input ingester to enable (string value)
+#ingester = drydock_provisioner.ingester.plugins.yaml.YamlIngester
+
+# List of module path strings of OOB drivers to enable (list value)
+#oob_driver = drydock_provisioner.drivers.oob.pyghmi_driver.PyghmiDriver
+
+# Module path string of the Node driver to enable (string value)
+#node_driver = drydock_provisioner.drivers.node.maasdriver.driver.MaasNodeDriver
+
+# Module path string of the Network driver enable (string value)
+#network_driver = <None>
+
+
+[pyghmi_driver]
+
+#
+# From drydock_provisioner
+#
+
+# Polling interval in seconds for querying IPMI status (integer value)
+#poll_interval = 10
+
+
+[timeouts]
+
+#
+# From drydock_provisioner
+#
+
+# Fallback timeout when a specific one is not configured (integer value)
+#drydock_timeout = 5
+
+# Timeout in minutes for creating site network templates (integer value)
+#create_network_template = 2
+
+# Timeout in minutes for creating user credentials (integer value)
+#configure_user_credentials = 2
+
+# Timeout in minutes for initial node identification (integer value)
+#identify_node = 10
+
+# Timeout in minutes for node commissioning and hardware configuration (integer
+# value)
+#configure_hardware = 30
+
+# Timeout in minutes for configuring node networking (integer value)
+#apply_node_networking = 5
+
+# Timeout in minutes for configuring node storage (integer value)
+#apply_node_storage = 5
+
+# Timeout in minutes for configuring node platform (integer value)
+#apply_node_platform = 5
+
+# Timeout in minutes for deploying a node (integer value)
+#deploy_node = 45
+
+# Timeout in minutes between deployment completion and the all boot actions
+# reporting status (integer value)
+#bootaction_final_status = 15

docs/source/_static/policy.yaml.sample

@@ -0,0 +1,62 @@
+# Actions requiring admin authority
+#"admin_required": "role:admin or is_admin:1"
+
+# Get task status
+# GET  /api/v1.0/tasks
+# GET  /api/v1.0/tasks/{task_id}
+#"physical_provisioner:read_task": "role:admin"
+
+# Create a task
+# POST  /api/v1.0/tasks
+#"physical_provisioner:create_task": "role:admin"
+
+# Create validate_design task
+# POST  /api/v1.0/tasks
+#"physical_provisioner:validate_design": "role:admin"
+
+# Create verify_site task
+# POST  /api/v1.0/tasks
+#"physical_provisioner:verify_site": "role:admin"
+
+# Create prepare_site task
+# POST  /api/v1.0/tasks
+#"physical_provisioner:prepare_site": "role:admin"
+
+# Create verify_nodes task
+# POST  /api/v1.0/tasks
+#"physical_provisioner:verify_nodes": "role:admin"
+
+# Create prepare_nodes task
+# POST  /api/v1.0/tasks
+#"physical_provisioner:prepare_nodes": "role:admin"
+
+# Create deploy_nodes task
+# POST  /api/v1.0/tasks
+#"physical_provisioner:deploy_nodes": "role:admin"
+
+# Create destroy_nodes task
+# POST  /api/v1.0/tasks
+#"physical_provisioner:destroy_nodes": "role:admin"
+
+# Read build data for a node
+# GET  /api/v1.0/nodes/{nodename}/builddata
+#"physical_provisioner:read_build_data": "role:admin"
+
+# Read loaded design data
+# GET  /api/v1.0/designs
+# GET  /api/v1.0/designs/{design_id}
+#"physical_provisioner:read_data": "role:admin"
+
+# Load design data
+# POST  /api/v1.0/designs
+# POST  /api/v1.0/designs/{design_id}/parts
+#"physical_provisioner:ingest_data": "role:admin"
+
+# et health status
+# GET  /api/v1.0/health/extended
+#"physical_provisioner:health_data": "role:admin"
+
+# Validate site design
+# POST  /api/v1.0/validatedesign
+#"physical_provisioner:validate_site_design": "role:admin"
+

drydock_provisioner/config.py

@@ -96,6 +96,22 @@ class DrydockConfig(object):
             'pool_size',
             default=15,
             help='The SQLalchemy database connection pool size.'),
+        cfg.BoolOpt(
+            'pool_pre_ping',
+            default=True,
+            help='Should DB connections be validated prior to use.'),
+        cfg.IntOpt(
+            'pool_timeout',
+            default=30,
+            help='How long a request for a connection should wait before one becomes available.'),
+        cfg.IntOpt(
+            'pool_overflow',
+            default=10,
+            help='How many connections above pool_size are allowed to be open during high usage.'),
+        cfg.IntOpt(
+            'connection_recycle',
+            default=-1,
+            help='Time, in seconds, when a connection should be closed and re-established. -1 for no recycling.'),
     ]
 
     # Options for the boot action framework

drydock_provisioner/statemgmt/state.py

@@ -43,7 +43,11 @@ class DrydockState(object):
         """Connect the state manager to the persistent DB."""
         self.db_engine = create_engine(
             config.config_mgr.conf.database.database_connect_string,
-            pool_size=config.config_mgr.conf.database.pool_size)
+            pool_size=config.config_mgr.conf.database.pool_size,
+            pool_pre_ping=config.config_mgr.conf.database.pool_pre_ping,
+            max_overflow=config.config_mgr.conf.database.pool_overflow,
+            pool_timeout=config.config_mgr.conf.database.pool_timeout,
+            pool_recycle=config.config_mgr.conf.database.connection_recycle)
         self.db_metadata = MetaData(bind=self.db_engine)
 
         self.tasks_tbl = tables.Tasks(self.db_metadata)

etc/drydock/drydock.conf.sample

@@ -29,6 +29,20 @@
 # The SQLalchemy database connection pool size. (integer value)
 #pool_size = 15
 
+# Should DB connections be validated prior to use. (boolean value)
+#pool_pre_ping = true
+
+# How long a request for a connection should wait before one becomes available.
+# (integer value)
+#pool_timeout = 30
+
+# How many connections above pool_size are allowed to be open during high usage.
+# (integer value)
+#pool_overflow = 10
+
+# Time, in seconds, when a connection should be closed and re-established. -1
+# for no recycling. (integer value)
+#connection_recycle = -1
 
 [keystone_authtoken]
 

requirements-direct.txt

@@ -16,7 +16,7 @@ oslo.policy==1.22.1
 iso8601==0.1.11
 keystoneauth1==3.3.0
 alembic==0.8.2
-sqlalchemy==1.1.14
+sqlalchemy==1.2.8
 psycopg2==2.7.3.1
 jsonschema==2.6.0
 jinja2==2.9.6

requirements-lock.txt

@@ -1,6 +1,6 @@
 alembic==0.8.2
-amqp==2.2.2
-Babel==2.5.3
+amqp==2.3.2
+Babel==2.6.0
 Beaker==1.9.1
 cachetools==2.1.0
 certifi==2018.4.16
@@ -14,13 +14,13 @@ falcon==1.4.1
 fasteners==0.14.1
 futurist==1.7.0
 greenlet==0.4.13
-idna==2.6
+idna==2.7
 iso8601==0.1.11
 Jinja2==2.9.6
 jsonschema==2.6.0
 keystoneauth1==3.3.0
 keystonemiddleware==4.9.1
-kombu==4.2.0
+kombu==4.2.1
 libvirt-python==3.10.0
 Mako==1.0.7
 MarkupSafe==1.0
@@ -31,19 +31,19 @@ netifaces==0.10.7
 oauthlib==2.1.0
 oslo.concurrency==3.27.0
 oslo.config==5.2.0
-oslo.context==2.20.0
+oslo.context==2.21.0
 oslo.i18n==3.20.0
-oslo.log==3.38.1
-oslo.messaging==6.2.0
+oslo.log==3.39.0
+oslo.messaging==6.5.0
 oslo.middleware==3.35.0
 oslo.policy==1.22.1
-oslo.serialization==2.25.0
+oslo.serialization==2.27.0
 oslo.service==1.31.2
-oslo.utils==3.36.2
+oslo.utils==3.36.3
 oslo.versionedobjects==1.23.0
 Paste==2.0.3
 PasteDeploy==1.5.2
-pbr==4.0.3
+pbr==4.0.4
 pip==10.0.1
 positional==1.2.1
 prettytable==0.7.2
@@ -56,24 +56,24 @@ pymongo==3.6.1
 pyparsing==2.2.0
 python-dateutil==2.7.3
 python-editor==1.0.3
-python-keystoneclient==3.16.0
+python-keystoneclient==3.17.0
 python-mimeparse==1.6.0
 pytz==2018.4
 PyYAML==3.12
 repoze.lru==0.7
-requests==2.18.4
+requests==2.19.1
 rfc3986==1.1.0
 Routes==2.4.1
 setuptools==39.2.0
 six==1.11.0
-SQLAlchemy==1.1.14
+SQLAlchemy==1.2.8
 statsd==3.2.2
 stevedore==1.28.0
 tenacity==4.12.0
 ulid2==0.1.1
-urllib3==1.22
+urllib3==1.23
 uWSGI==2.0.15
 vine==1.1.4
-WebOb==1.8.1
+WebOb==1.8.2
 wheel==0.31.1
 wrapt==1.10.11