From d9567272a5c180c52af431c73b015a6120b8ba78 Mon Sep 17 00:00:00 2001 From: David Moreau Simard Date: Fri, 26 Feb 2016 14:41:32 -0500 Subject: [PATCH] Add --service-workers parameter and scale mariadb connections This is a new config that will define the value to be used in parameters passed to different services as an amount of workers or threads to use. Most puppet modules default the amount of workers or threads to $::processorcount and this yields fairly high RAM usage on machines with a lot of cores. Since Packstack did not set this, we would use the default provided by the module. 8 cores with fairly minimal services yields >8GB RAM usage after tempest tests. If we set --service-workers to 2, we peak at around 6.5GB RAM for the same Packstack parameters. We will still default to $::processorcount for backwards compatibility but this new parameter can be used to lower the RAM usage where there are constraints (i.e, upstream gate) Additionally, instead of hardcoding a max_connections of 1024 which provides a ceiling of ~3GB RAM, we now instead default to a sane value depending on the amount of service workers (n*128). This will still yield the same 1024 max_connections for 8 cores but would set a max_connection of 256 when using 2 service workers for example. Change-Id: I2153ce522f227678d5318080dfb4ef171b36cbf7 --- docs/packstack.rst | 3 +++ packstack/plugins/prescript_000.py | 14 ++++++++++++++ packstack/puppet/templates/aodh.pp | 3 ++- packstack/puppet/templates/ceilometer.pp | 1 + packstack/puppet/templates/cinder.pp | 3 ++- packstack/puppet/templates/glance.pp | 2 ++ packstack/puppet/templates/global.pp | 2 ++ packstack/puppet/templates/gnocchi.pp | 3 ++- packstack/puppet/templates/heat_cfn.pp | 4 +++- packstack/puppet/templates/heat_cloudwatch.pp | 4 +++- packstack/puppet/templates/keystone.pp | 3 ++- packstack/puppet/templates/mariadb_install.pp | 4 +++- packstack/puppet/templates/neutron_api.pp | 2 ++ packstack/puppet/templates/neutron_metadata.pp | 13 +++++++------ packstack/puppet/templates/nova_api.pp | 4 +++- packstack/puppet/templates/sahara.pp | 4 +++- packstack/puppet/templates/swift_proxy.pp | 1 + packstack/puppet/templates/trove.pp | 1 + 18 files changed, 56 insertions(+), 15 deletions(-) diff --git a/docs/packstack.rst b/docs/packstack.rst index bd86e1761..4fa7d6fbb 100644 --- a/docs/packstack.rst +++ b/docs/packstack.rst @@ -32,6 +32,9 @@ Global Options **CONFIG_DEFAULT_PASSWORD** Default password to be used everywhere (overridden by passwords set for individual services or users). +**CONFIG_SERVICE_WORKERS** + The amount of service workers/threads to use for each service. Useful to tweak when you have memory constraints. Defaults to the amount of cores on the system. + **CONFIG_MARIADB_INSTALL** Specify 'y' to install MariaDB. ['y', 'n'] diff --git a/packstack/plugins/prescript_000.py b/packstack/plugins/prescript_000.py index ac795d914..82182f979 100644 --- a/packstack/plugins/prescript_000.py +++ b/packstack/plugins/prescript_000.py @@ -79,6 +79,20 @@ def initConfig(controller): "NEED_CONFIRM": True, "CONDITION": False}, + {"CMD_OPTION": "service-workers", + "PROMPT": ( + "Enter the amount of service workers/threads to use for each " + "service. Leave blank to use the default." + ), + "OPTION_LIST": [], + "DEFAULT_VALUE": '%{::processorcount}', + "MASK_INPUT": False, + "LOOSE_VALIDATION": False, + "CONF_NAME": "CONFIG_SERVICE_WORKERS", + "USE_DEFAULT": False, + "NEED_CONFIRM": True, + "CONDITION": False}, + {"CMD_OPTION": "mariadb-install", "PROMPT": "Should Packstack install MariaDB", "OPTION_LIST": ["y", "n"], diff --git a/packstack/puppet/templates/aodh.pp b/packstack/puppet/templates/aodh.pp index 016f9026c..d44afe2e5 100644 --- a/packstack/puppet/templates/aodh.pp +++ b/packstack/puppet/templates/aodh.pp @@ -35,7 +35,8 @@ class { '::apache': } class { '::aodh::wsgi::apache': - ssl => false, + workers => $service_workers, + ssl => false } if hiera('CONFIG_KEYSTONE_SERVICE_NAME') == 'httpd' { apache::listen { '5000': } diff --git a/packstack/puppet/templates/ceilometer.pp b/packstack/puppet/templates/ceilometer.pp index 19867056d..4dad64a8d 100644 --- a/packstack/puppet/templates/ceilometer.pp +++ b/packstack/puppet/templates/ceilometer.pp @@ -56,4 +56,5 @@ class { '::ceilometer::api': keystone_auth_uri => hiera('CONFIG_KEYSTONE_PUBLIC_URL'), keystone_identity_uri => hiera('CONFIG_KEYSTONE_ADMIN_URL'), keystone_password => hiera('CONFIG_CEILOMETER_KS_PW'), + api_workers => $service_workers } diff --git a/packstack/puppet/templates/cinder.pp b/packstack/puppet/templates/cinder.pp index 7170dd361..45bea058e 100644 --- a/packstack/puppet/templates/cinder.pp +++ b/packstack/puppet/templates/cinder.pp @@ -16,7 +16,8 @@ class { '::cinder::api': auth_uri => hiera('CONFIG_KEYSTONE_PUBLIC_URL'), identity_uri => hiera('CONFIG_KEYSTONE_ADMIN_URL'), nova_catalog_info => 'compute:nova:publicURL', - nova_catalog_admin_info => 'compute:nova:adminURL' + nova_catalog_admin_info => 'compute:nova:adminURL', + service_workers => $service_workers } class { '::cinder::scheduler': } diff --git a/packstack/puppet/templates/glance.pp b/packstack/puppet/templates/glance.pp index 874f16840..e5c767e15 100644 --- a/packstack/puppet/templates/glance.pp +++ b/packstack/puppet/templates/glance.pp @@ -28,6 +28,7 @@ class { '::glance::api': verbose => true, debug => hiera('CONFIG_DEBUG_MODE'), os_region_name => hiera('CONFIG_KEYSTONE_REGION'), + workers => $service_workers } class { '::glance::registry': @@ -40,4 +41,5 @@ class { '::glance::registry': database_connection => "mysql+pymysql://glance:${glance_ks_pw}@${glance_mariadb_host}/glance", verbose => true, debug => hiera('CONFIG_DEBUG_MODE'), + workers => $service_workers } diff --git a/packstack/puppet/templates/global.pp b/packstack/puppet/templates/global.pp index fb623a137..fea892641 100644 --- a/packstack/puppet/templates/global.pp +++ b/packstack/puppet/templates/global.pp @@ -5,4 +5,6 @@ $use_subnets = $use_subnets_value ? { default => false, } +$service_workers = hiera('CONFIG_SERVICE_WORKERS') + Exec { timeout => hiera('DEFAULT_EXEC_TIMEOUT') } diff --git a/packstack/puppet/templates/gnocchi.pp b/packstack/puppet/templates/gnocchi.pp index a8fc58a8e..7d33f4666 100644 --- a/packstack/puppet/templates/gnocchi.pp +++ b/packstack/puppet/templates/gnocchi.pp @@ -6,7 +6,8 @@ class { '::apache': } class { '::gnocchi::wsgi::apache': - ssl => false, + workers => $service_workers, + ssl => false } if hiera('CONFIG_KEYSTONE_SERVICE_NAME') == 'httpd' { apache::listen { '5000': } diff --git a/packstack/puppet/templates/heat_cfn.pp b/packstack/puppet/templates/heat_cfn.pp index 66d529185..6aa5961fe 100644 --- a/packstack/puppet/templates/heat_cfn.pp +++ b/packstack/puppet/templates/heat_cfn.pp @@ -1,5 +1,7 @@ -class { '::heat::api_cfn': } +class { '::heat::api_cfn': + workers => $service_workers +} $heat_cfn_cfg_ctrl_host = hiera('CONFIG_KEYSTONE_HOST_URL') diff --git a/packstack/puppet/templates/heat_cloudwatch.pp b/packstack/puppet/templates/heat_cloudwatch.pp index d0de4a30c..3cf7e0696 100644 --- a/packstack/puppet/templates/heat_cloudwatch.pp +++ b/packstack/puppet/templates/heat_cloudwatch.pp @@ -1,3 +1,5 @@ -class { '::heat::api_cloudwatch': } +class { '::heat::api_cloudwatch': + workers => $service_workers +} diff --git a/packstack/puppet/templates/keystone.pp b/packstack/puppet/templates/keystone.pp index 870f00427..da8d05114 100644 --- a/packstack/puppet/templates/keystone.pp +++ b/packstack/puppet/templates/keystone.pp @@ -50,7 +50,8 @@ if $keystone_service_name == 'httpd' { } class { '::keystone::wsgi::apache': - ssl => $keystone_use_ssl, + workers => $service_workers, + ssl => $keystone_use_ssl } if hiera('CONFIG_HORIZON_SSL') == 'y' { diff --git a/packstack/puppet/templates/mariadb_install.pp b/packstack/puppet/templates/mariadb_install.pp index ac2bd4c72..2a001e3cd 100644 --- a/packstack/puppet/templates/mariadb_install.pp +++ b/packstack/puppet/templates/mariadb_install.pp @@ -1,4 +1,6 @@ +$max_connections = $service_workers * 128 + # Package mariadb-server conflicts with mariadb-galera-server package { 'mariadb-server': ensure => absent, @@ -21,7 +23,7 @@ class { '::mysql::server': 'mysqld' => { 'bind_address' => $bind_address, 'default_storage_engine' => 'InnoDB', - 'max_connections' => '1024', + 'max_connections' => $max_connections, 'open_files_limit' => '-1', # galera options 'wsrep_provider' => 'none', diff --git a/packstack/puppet/templates/neutron_api.pp b/packstack/puppet/templates/neutron_api.pp index cf5a0b0f8..6f3e6027d 100644 --- a/packstack/puppet/templates/neutron_api.pp +++ b/packstack/puppet/templates/neutron_api.pp @@ -6,6 +6,8 @@ class { '::neutron::server': identity_uri => hiera('CONFIG_KEYSTONE_ADMIN_URL'), sync_db => true, enabled => true, + api_workers => $service_workers, + rpc_workers => $service_workers } # TODO: FIXME: remove this hack after upstream resolves https://bugs.launchpad.net/puppet-neutron/+bug/1474961 diff --git a/packstack/puppet/templates/neutron_metadata.pp b/packstack/puppet/templates/neutron_metadata.pp index 9205c49fd..e4dd99ca2 100644 --- a/packstack/puppet/templates/neutron_metadata.pp +++ b/packstack/puppet/templates/neutron_metadata.pp @@ -1,8 +1,9 @@ class { '::neutron::agents::metadata': - auth_password => hiera('CONFIG_NEUTRON_KS_PW'), - auth_url => hiera('CONFIG_KEYSTONE_PUBLIC_URL'), - auth_region => hiera('CONFIG_KEYSTONE_REGION'), - shared_secret => hiera('CONFIG_NEUTRON_METADATA_PW'), - metadata_ip => force_ip(hiera('CONFIG_KEYSTONE_HOST_URL')), - debug => hiera('CONFIG_DEBUG_MODE'), + auth_password => hiera('CONFIG_NEUTRON_KS_PW'), + auth_url => hiera('CONFIG_KEYSTONE_PUBLIC_URL'), + auth_region => hiera('CONFIG_KEYSTONE_REGION'), + shared_secret => hiera('CONFIG_NEUTRON_METADATA_PW'), + metadata_ip => force_ip(hiera('CONFIG_KEYSTONE_HOST_URL')), + debug => hiera('CONFIG_DEBUG_MODE'), + metadata_workers => $service_workers } diff --git a/packstack/puppet/templates/nova_api.pp b/packstack/puppet/templates/nova_api.pp index 888138991..cbba853f5 100644 --- a/packstack/puppet/templates/nova_api.pp +++ b/packstack/puppet/templates/nova_api.pp @@ -23,7 +23,9 @@ class { '::nova::api': neutron_metadata_proxy_shared_secret => hiera('CONFIG_NEUTRON_METADATA_PW_UNQUOTED', undef), default_floating_pool => $default_floating_pool, pci_alias => hiera('CONFIG_NOVA_PCI_ALIAS'), - sync_db_api => true + sync_db_api => true, + osapi_compute_workers => $service_workers, + metadata_workers => $service_workers } Package<| title == 'nova-common' |> -> Class['nova::api'] diff --git a/packstack/puppet/templates/sahara.pp b/packstack/puppet/templates/sahara.pp index 26037da0e..16bc813c9 100644 --- a/packstack/puppet/templates/sahara.pp +++ b/packstack/puppet/templates/sahara.pp @@ -1,3 +1,5 @@ -class { '::sahara::service::api': } +class { '::sahara::service::api': + api_workers => $service_workers +} class { '::sahara::service::engine': } diff --git a/packstack/puppet/templates/swift_proxy.pp b/packstack/puppet/templates/swift_proxy.pp index 6e20cdc0c..35fa55535 100644 --- a/packstack/puppet/templates/swift_proxy.pp +++ b/packstack/puppet/templates/swift_proxy.pp @@ -41,6 +41,7 @@ class { '::swift::proxy': 'proxy-server', ], account_autocreate => true, + workers => $service_workers } # configure all of the middlewares diff --git a/packstack/puppet/templates/trove.pp b/packstack/puppet/templates/trove.pp index 851cc6683..92e6efceb 100644 --- a/packstack/puppet/templates/trove.pp +++ b/packstack/puppet/templates/trove.pp @@ -14,6 +14,7 @@ class { '::trove::api': ca_file => false, verbose => true, debug => hiera('CONFIG_DEBUG_MODE'), + workers => $service_workers } class { '::trove::conductor':