Add --service-workers parameter and scale mariadb connections

This is a new config that will define the value to be used
in parameters passed to different services as an amount of
workers or threads to use.
Most puppet modules default the amount of workers or threads
to $::processorcount and this yields fairly high RAM usage on
machines with a lot of cores. Since Packstack did not set this,
we would use the default provided by the module.

8 cores with fairly minimal services yields >8GB RAM usage after
tempest tests.
If we set --service-workers to 2, we peak at around 6.5GB RAM for
the same Packstack parameters.

We will still default to $::processorcount for backwards
compatibility but this new parameter can be used to lower the
RAM usage where there are constraints (i.e, upstream gate)

Additionally, instead of hardcoding a max_connections of 1024
which provides a ceiling of ~3GB RAM, we now instead default to
a sane value depending on the amount of service workers (n*128).
This will still yield the same 1024 max_connections for 8 cores
but would set a max_connection of 256 when using 2 service workers
for example.

Change-Id: I2153ce522f227678d5318080dfb4ef171b36cbf7

docs/packstack.rst

@@ -32,6 +32,9 @@ Global Options
 **CONFIG_DEFAULT_PASSWORD**
     Default password to be used everywhere (overridden by passwords set for individual services or users).
 
+**CONFIG_SERVICE_WORKERS**
+    The amount of service workers/threads to use for each service. Useful to tweak when you have memory constraints. Defaults to the amount of cores on the system.
+
 **CONFIG_MARIADB_INSTALL**
     Specify 'y' to install MariaDB. ['y', 'n']
 

packstack/plugins/prescript_000.py

@@ -79,6 +79,20 @@ def initConfig(controller):
              "NEED_CONFIRM": True,
              "CONDITION": False},
 
+            {"CMD_OPTION": "service-workers",
+             "PROMPT": (
+                 "Enter the amount of service workers/threads to use for each "
+                 "service. Leave blank to use the default."
+             ),
+             "OPTION_LIST": [],
+             "DEFAULT_VALUE": '%{::processorcount}',
+             "MASK_INPUT": False,
+             "LOOSE_VALIDATION": False,
+             "CONF_NAME": "CONFIG_SERVICE_WORKERS",
+             "USE_DEFAULT": False,
+             "NEED_CONFIRM": True,
+             "CONDITION": False},
+
             {"CMD_OPTION": "mariadb-install",
              "PROMPT": "Should Packstack install MariaDB",
              "OPTION_LIST": ["y", "n"],

packstack/puppet/templates/aodh.pp

@@ -35,7 +35,8 @@ class { '::apache':
 }
 
 class { '::aodh::wsgi::apache':
-  ssl => false,
+  workers => $service_workers,
+  ssl     => false
 }
 if hiera('CONFIG_KEYSTONE_SERVICE_NAME') == 'httpd' {
   apache::listen { '5000': }

packstack/puppet/templates/ceilometer.pp

@@ -56,4 +56,5 @@ class { '::ceilometer::api':
   keystone_auth_uri     => hiera('CONFIG_KEYSTONE_PUBLIC_URL'),
   keystone_identity_uri => hiera('CONFIG_KEYSTONE_ADMIN_URL'),
   keystone_password     => hiera('CONFIG_CEILOMETER_KS_PW'),
+  api_workers           => $service_workers
 }

packstack/puppet/templates/cinder.pp

@@ -16,7 +16,8 @@ class { '::cinder::api':
   auth_uri                => hiera('CONFIG_KEYSTONE_PUBLIC_URL'),
   identity_uri            => hiera('CONFIG_KEYSTONE_ADMIN_URL'),
   nova_catalog_info       => 'compute:nova:publicURL',
-  nova_catalog_admin_info => 'compute:nova:adminURL'
+  nova_catalog_admin_info => 'compute:nova:adminURL',
+  service_workers         => $service_workers
 }
 
 class { '::cinder::scheduler': }

packstack/puppet/templates/glance.pp

@@ -28,6 +28,7 @@ class { '::glance::api':
   verbose             => true,
   debug               => hiera('CONFIG_DEBUG_MODE'),
   os_region_name      => hiera('CONFIG_KEYSTONE_REGION'),
+  workers             => $service_workers
 }
 
 class { '::glance::registry':
@@ -40,4 +41,5 @@ class { '::glance::registry':
   database_connection => "mysql+pymysql://glance:${glance_ks_pw}@${glance_mariadb_host}/glance",
   verbose             => true,
   debug               => hiera('CONFIG_DEBUG_MODE'),
+  workers             => $service_workers
 }

packstack/puppet/templates/global.pp

@@ -5,4 +5,6 @@ $use_subnets = $use_subnets_value ? {
   default => false,
 }
 
+$service_workers = hiera('CONFIG_SERVICE_WORKERS')
+
 Exec { timeout => hiera('DEFAULT_EXEC_TIMEOUT') }

packstack/puppet/templates/gnocchi.pp

@@ -6,7 +6,8 @@ class { '::apache':
 }
 
 class { '::gnocchi::wsgi::apache':
-  ssl => false,
+  workers => $service_workers,
+  ssl     => false
 }
 if hiera('CONFIG_KEYSTONE_SERVICE_NAME') == 'httpd' {
   apache::listen { '5000': }

packstack/puppet/templates/heat_cfn.pp

@@ -1,5 +1,7 @@
 
-class { '::heat::api_cfn': }
+class { '::heat::api_cfn':
+  workers => $service_workers
+}
 
 $heat_cfn_cfg_ctrl_host = hiera('CONFIG_KEYSTONE_HOST_URL')
 

packstack/puppet/templates/heat_cloudwatch.pp

@@ -1,3 +1,5 @@
 
-class { '::heat::api_cloudwatch': }
+class { '::heat::api_cloudwatch':
+  workers => $service_workers
+}
 

packstack/puppet/templates/keystone.pp

@@ -50,7 +50,8 @@ if $keystone_service_name == 'httpd' {
   }
 
   class { '::keystone::wsgi::apache':
-    ssl => $keystone_use_ssl,
+    workers => $service_workers,
+    ssl     => $keystone_use_ssl
   }
 
   if hiera('CONFIG_HORIZON_SSL') == 'y' {

packstack/puppet/templates/mariadb_install.pp

@@ -1,4 +1,6 @@
 
+$max_connections = $service_workers * 128
+
 # Package mariadb-server conflicts with mariadb-galera-server
 package { 'mariadb-server':
   ensure => absent,
@@ -21,7 +23,7 @@ class { '::mysql::server':
     'mysqld' => {
       'bind_address'           => $bind_address,
       'default_storage_engine' => 'InnoDB',
-      'max_connections'        => '1024',
+      'max_connections'        => $max_connections,
       'open_files_limit'       => '-1',
       # galera options
       'wsrep_provider'         => 'none',

packstack/puppet/templates/neutron_api.pp

@@ -6,6 +6,8 @@ class { '::neutron::server':
   identity_uri        => hiera('CONFIG_KEYSTONE_ADMIN_URL'),
   sync_db             => true,
   enabled             => true,
+  api_workers         => $service_workers,
+  rpc_workers         => $service_workers
 }
 
 # TODO: FIXME: remove this hack after upstream resolves https://bugs.launchpad.net/puppet-neutron/+bug/1474961

packstack/puppet/templates/neutron_metadata.pp

@@ -1,8 +1,9 @@
 class { '::neutron::agents::metadata':
-  auth_password => hiera('CONFIG_NEUTRON_KS_PW'),
+  auth_password    => hiera('CONFIG_NEUTRON_KS_PW'),
-  auth_url      => hiera('CONFIG_KEYSTONE_PUBLIC_URL'),
+  auth_url         => hiera('CONFIG_KEYSTONE_PUBLIC_URL'),
-  auth_region   => hiera('CONFIG_KEYSTONE_REGION'),
+  auth_region      => hiera('CONFIG_KEYSTONE_REGION'),
-  shared_secret => hiera('CONFIG_NEUTRON_METADATA_PW'),
+  shared_secret    => hiera('CONFIG_NEUTRON_METADATA_PW'),
-  metadata_ip   => force_ip(hiera('CONFIG_KEYSTONE_HOST_URL')),
+  metadata_ip      => force_ip(hiera('CONFIG_KEYSTONE_HOST_URL')),
-  debug         => hiera('CONFIG_DEBUG_MODE'),
+  debug            => hiera('CONFIG_DEBUG_MODE'),
+  metadata_workers => $service_workers
 }

packstack/puppet/templates/nova_api.pp

@@ -23,7 +23,9 @@ class { '::nova::api':
   neutron_metadata_proxy_shared_secret => hiera('CONFIG_NEUTRON_METADATA_PW_UNQUOTED', undef),
   default_floating_pool                => $default_floating_pool,
   pci_alias                            => hiera('CONFIG_NOVA_PCI_ALIAS'),
-  sync_db_api                          => true
+  sync_db_api                          => true,
+  osapi_compute_workers                => $service_workers,
+  metadata_workers                     => $service_workers
 }
 
 Package<| title == 'nova-common' |> -> Class['nova::api']

packstack/puppet/templates/sahara.pp

@@ -1,3 +1,5 @@
-class { '::sahara::service::api': }
+class { '::sahara::service::api':
+  api_workers => $service_workers
+}
 
 class { '::sahara::service::engine': }

packstack/puppet/templates/swift_proxy.pp

@@ -41,6 +41,7 @@ class { '::swift::proxy':
     'proxy-server',
   ],
   account_autocreate => true,
+  workers => $service_workers
 }
 
 # configure all of the middlewares

packstack/puppet/templates/trove.pp

@@ -14,6 +14,7 @@ class { '::trove::api':
   ca_file           => false,
   verbose           => true,
   debug             => hiera('CONFIG_DEBUG_MODE'),
+  workers           => $service_workers
 }
 
 class { '::trove::conductor':