openstack/devstack
Code Issues Proposed changes
ea774b684d
devstack/lib/tls
Clark Boylan cfb9f057ea Tune apache connections for tls proxy 
We are seeing connection errors to the proxy occasionally. These errors
do not result in a logged http request or error to the backends,
resulting in a theory that the proxy itself may just not be able to
handle the number of connections. More than double the total number of
connections that will be accepted by the proxy in an attempt to fix
this.

Change-Id: Iefa6c43451dd1f95927528d2ce0003c84248847f
Related-bug: 1630664
2016-11-29 10:43:05 -08:00

581 lines
17 KiB
Bash
Raw Blame History

#!/bin/bash
#
# lib/tls
# Functions to control the configuration and operation of the TLS proxy service
# !! source _before_ any services that use ``SERVICE_HOST``
#
# Dependencies:
#
# - ``functions`` file
# - ``DEST``, ``DATA_DIR`` must be defined
# - ``HOST_IP``, ``SERVICE_HOST``
# - ``KEYSTONE_TOKEN_FORMAT`` must be defined
# Entry points:
#
# - configure_CA
# - init_CA
# - configure_proxy
# - start_tls_proxy
# - stop_tls_proxy
# - cleanup_CA
# - make_root_CA
# - make_int_CA
# - make_cert ca-dir cert-name "common-name" ["alt-name" ...]
# - start_tls_proxy HOST_IP 5000 localhost 5000
# - ensure_certificates
# - is_ssl_enabled_service
# - enable_mod_ssl
# Defaults
# --------
if is_service_enabled tls-proxy; then
# TODO(dtroyer): revisit this below after the search for HOST_IP has been done
TLS_IP=${TLS_IP:-$SERVICE_IP}
fi
DEVSTACK_HOSTNAME=$(hostname -f)
DEVSTACK_CERT_NAME=devstack-cert
DEVSTACK_CERT=$DATA_DIR/$DEVSTACK_CERT_NAME.pem
# CA configuration
ROOT_CA_DIR=${ROOT_CA_DIR:-$DATA_DIR/CA/root-ca}
INT_CA_DIR=${INT_CA_DIR:-$DATA_DIR/CA/int-ca}
ORG_NAME="OpenStack"
ORG_UNIT_NAME="DevStack"
# Stud configuration
STUD_PROTO="--tls"
STUD_CIPHERS='TLSv1+HIGH:!DES:!aNULL:!eNULL:@STRENGTH'
# CA Functions
# ============
# There may be more than one, get specific
OPENSSL=${OPENSSL:-/usr/bin/openssl}
# Do primary CA configuration
function configure_CA {
# build common config file
# Verify ``TLS_IP`` is good
if [[ -n "$HOST_IP" && "$HOST_IP" != "$TLS_IP" ]]; then
# auto-discover has changed the IP
TLS_IP=$HOST_IP
fi
}
# Creates a new CA directory structure
# create_CA_base ca-dir
function create_CA_base {
local ca_dir=$1
if [[ -d $ca_dir ]]; then
# Bail out it exists
return 0
fi
local i
for i in certs crl newcerts private; do
mkdir -p $ca_dir/$i
done
chmod 710 $ca_dir/private
echo "01" >$ca_dir/serial
cp /dev/null $ca_dir/index.txt
}
# Create a new CA configuration file
# create_CA_config ca-dir common-name
function create_CA_config {
local ca_dir=$1
local common_name=$2
echo "
[ ca ]
default_ca = CA_default
[ CA_default ]
dir = $ca_dir
policy = policy_match
database = \$dir/index.txt
serial = \$dir/serial
certs = \$dir/certs
crl_dir = \$dir/crl
new_certs_dir = \$dir/newcerts
certificate = \$dir/cacert.pem
private_key = \$dir/private/cacert.key
RANDFILE = \$dir/private/.rand
default_md = default
[ req ]
default_bits = 1024
default_md = sha1
prompt = no
distinguished_name = ca_distinguished_name
x509_extensions = ca_extensions
[ ca_distinguished_name ]
organizationName = $ORG_NAME
organizationalUnitName = $ORG_UNIT_NAME Certificate Authority
commonName = $common_name
[ policy_match ]
countryName = optional
stateOrProvinceName = optional
organizationName = match
organizationalUnitName = optional
commonName = supplied
[ ca_extensions ]
basicConstraints = critical,CA:true
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always, issuer
keyUsage = cRLSign, keyCertSign
" >$ca_dir/ca.conf
}
# Create a new signing configuration file
# create_signing_config ca-dir
function create_signing_config {
local ca_dir=$1
echo "
[ ca ]
default_ca = CA_default
[ CA_default ]
dir = $ca_dir
policy = policy_match
database = \$dir/index.txt
serial = \$dir/serial
certs = \$dir/certs
crl_dir = \$dir/crl
new_certs_dir = \$dir/newcerts
certificate = \$dir/cacert.pem
private_key = \$dir/private/cacert.key
RANDFILE = \$dir/private/.rand
default_md = default
[ req ]
default_bits = 1024
default_md = sha1
prompt = no
distinguished_name = req_distinguished_name
x509_extensions = req_extensions
[ req_distinguished_name ]
organizationName = $ORG_NAME
organizationalUnitName = $ORG_UNIT_NAME Server Farm
[ policy_match ]
countryName = optional
stateOrProvinceName = optional
organizationName = match
organizationalUnitName = optional
commonName = supplied
[ req_extensions ]
basicConstraints = CA:false
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always, issuer
keyUsage = digitalSignature, keyEncipherment, keyAgreement
extendedKeyUsage = serverAuth, clientAuth
subjectAltName = \$ENV::SUBJECT_ALT_NAME
" >$ca_dir/signing.conf
}
# Create root and intermediate CAs
# init_CA
function init_CA {
# Ensure CAs are built
make_root_CA $ROOT_CA_DIR
make_int_CA $INT_CA_DIR $ROOT_CA_DIR
# Create the CA bundle
cat $ROOT_CA_DIR/cacert.pem $INT_CA_DIR/cacert.pem >>$INT_CA_DIR/ca-chain.pem
cat $INT_CA_DIR/ca-chain.pem >> $SSL_BUNDLE_FILE
if is_fedora; then
sudo cp $INT_CA_DIR/ca-chain.pem /usr/share/pki/ca-trust-source/anchors/devstack-chain.pem
sudo update-ca-trust
elif is_ubuntu; then
sudo cp $INT_CA_DIR/ca-chain.pem /usr/local/share/ca-certificates/devstack-int.crt
sudo cp $ROOT_CA_DIR/cacert.pem /usr/local/share/ca-certificates/devstack-root.crt
sudo update-ca-certificates
fi
}
# Create an initial server cert
# init_cert
function init_cert {
if [[ ! -r $DEVSTACK_CERT ]]; then
if [[ -n "$TLS_IP" ]]; then
# Lie to let incomplete match routines work
TLS_IP="DNS:$TLS_IP,IP:$TLS_IP"
fi
make_cert $INT_CA_DIR $DEVSTACK_CERT_NAME $DEVSTACK_HOSTNAME "$TLS_IP"
# Create a cert bundle
cat $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/cacert.pem >$DEVSTACK_CERT
fi
}
# make_cert creates and signs a new certificate with the given commonName and CA
# make_cert ca-dir cert-name "common-name" ["alt-name" ...]
function make_cert {
local ca_dir=$1
local cert_name=$2
local common_name=$3
local alt_names=$4
if [ "$common_name" != "$SERVICE_HOST" ]; then
if [[ -z "$alt_names" ]]; then
alt_names="DNS:$SERVICE_HOST"
else
alt_names="$alt_names,DNS:$SERVICE_HOST"
fi
if is_ipv4_address "$SERVICE_HOST" ; then
alt_names="$alt_names,IP:$SERVICE_HOST"
fi
fi
# Only generate the certificate if it doesn't exist yet on the disk
if [ ! -r "$ca_dir/$cert_name.crt" ]; then
# Generate a signing request
$OPENSSL req \
-sha1 \
-newkey rsa \
-nodes \
-keyout $ca_dir/private/$cert_name.key \
-out $ca_dir/$cert_name.csr \
-subj "/O=${ORG_NAME}/OU=${ORG_UNIT_NAME} Servers/CN=${common_name}"
if [[ -z "$alt_names" ]]; then
alt_names="DNS:${common_name}"
else
alt_names="DNS:${common_name},${alt_names}"
fi
# Sign the request valid for 1 year
SUBJECT_ALT_NAME="$alt_names" \
$OPENSSL ca -config $ca_dir/signing.conf \
-extensions req_extensions \
-days 365 \
-notext \
-in $ca_dir/$cert_name.csr \
-out $ca_dir/$cert_name.crt \
-subj "/O=${ORG_NAME}/OU=${ORG_UNIT_NAME} Servers/CN=${common_name}" \
-batch
fi
}
# Make an intermediate CA to sign everything else
# make_int_CA ca-dir signing-ca-dir
function make_int_CA {
local ca_dir=$1
local signing_ca_dir=$2
# Create the root CA
create_CA_base $ca_dir
create_CA_config $ca_dir 'Intermediate CA'
create_signing_config $ca_dir
if [ ! -r "$ca_dir/cacert.pem" ]; then
# Create a signing certificate request
$OPENSSL req -config $ca_dir/ca.conf \
-sha1 \
-newkey rsa \
-nodes \
-keyout $ca_dir/private/cacert.key \
-out $ca_dir/cacert.csr \
-outform PEM
# Sign the intermediate request valid for 1 year
$OPENSSL ca -config $signing_ca_dir/ca.conf \
-extensions ca_extensions \
-days 365 \
-notext \
-in $ca_dir/cacert.csr \
-out $ca_dir/cacert.pem \
-batch
fi
}
# Make a root CA to sign other CAs
# make_root_CA ca-dir
function make_root_CA {
local ca_dir=$1
# Create the root CA
create_CA_base $ca_dir
create_CA_config $ca_dir 'Root CA'
if [ ! -r "$ca_dir/cacert.pem" ]; then
# Create a self-signed certificate valid for 5 years
$OPENSSL req -config $ca_dir/ca.conf \
-x509 \
-nodes \
-newkey rsa \
-days 21360 \
-keyout $ca_dir/private/cacert.key \
-out $ca_dir/cacert.pem \
-outform PEM
fi
}
# If a non-system python-requests is installed then it will use the
# built-in CA certificate store rather than the distro-specific
# CA certificate store. Detect this and symlink to the correct
# one. If the value for the CA is not rooted in /etc then we know
# we need to change it.
function fix_system_ca_bundle_path {
if is_service_enabled tls-proxy || [ "$USE_SSL" == "True" ]; then
local capath
capath=$(python -c $'try:\n from requests import certs\n print certs.where()\nexcept ImportError: pass')
if [[ ! $capath == "" && ! $capath =~ ^/etc/.* && ! -L $capath ]]; then
if is_fedora; then
sudo rm -f $capath
sudo ln -s /etc/pki/tls/certs/ca-bundle.crt $capath
elif is_ubuntu; then
sudo rm -f $capath
sudo ln -s /etc/ssl/certs/ca-certificates.crt $capath
else
echo "Don't know how to set the CA bundle, expect the install to fail."
fi
fi
fi
}
# Certificate Input Configuration
# ===============================
# check to see if the service(s) specified are to be SSL enabled.
#
# Multiple services specified as arguments are ``OR``'ed together; the test
# is a short-circuit boolean, i.e it returns on the first match.
#
# Uses global ``SSL_ENABLED_SERVICES``
function is_ssl_enabled_service {
local services=$@
local service=""
if [ "$USE_SSL" == "False" ]; then
return 1
fi
for service in ${services}; do
[[ ,${SSL_ENABLED_SERVICES}, =~ ,${service}, ]] && return 0
done
return 1
}
# Ensure that the certificates for a service are in place. This function does
# not check that a service is SSL enabled, this should already have been
# completed.
#
# The function expects to find a certificate, key and CA certificate in the
# variables ``{service}_SSL_CERT``, ``{service}_SSL_KEY`` and ``{service}_SSL_CA``. For
# example for keystone this would be ``KEYSTONE_SSL_CERT``, ``KEYSTONE_SSL_KEY`` and
# ``KEYSTONE_SSL_CA``.
#
# If it does not find these certificates then the DevStack-issued server
# certificate, key and CA certificate will be associated with the service.
#
# If only some of the variables are provided then the function will quit.
function ensure_certificates {
local service=$1
local cert_var="${service}_SSL_CERT"
local key_var="${service}_SSL_KEY"
local ca_var="${service}_SSL_CA"
local cert=${!cert_var}
local key=${!key_var}
local ca=${!ca_var}
if [[ -z "$cert" && -z "$key" && -z "$ca" ]]; then
local cert="$INT_CA_DIR/$DEVSTACK_CERT_NAME.crt"
local key="$INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key"
local ca="$INT_CA_DIR/ca-chain.pem"
eval ${service}_SSL_CERT=\$cert
eval ${service}_SSL_KEY=\$key
eval ${service}_SSL_CA=\$ca
return # the CA certificate is already in the bundle
elif [[ -z "$cert" || -z "$key" || -z "$ca" ]]; then
die $LINENO "Missing either the ${cert_var} ${key_var} or ${ca_var}" \
"variable to enable SSL for ${service}"
fi
cat $ca >> $SSL_BUNDLE_FILE
}
# Enable the mod_ssl plugin in Apache
function enable_mod_ssl {
echo "Enabling mod_ssl"
if is_ubuntu; then
sudo a2enmod ssl
elif is_fedora; then
# Fedora enables mod_ssl by default
:
fi
if ! sudo `which httpd || which apache2ctl` -M | grep -w -q ssl_module; then
die $LINENO "mod_ssl is not enabled in apache2/httpd, please check for it manually and run stack.sh again"
fi
}
# Proxy Functions
# ===============
function tune_apache_connections {
local tuning_file=$APACHE_SETTINGS_DIR/connection-tuning.conf
if ! [ -f $tuning_file ] ; then
sudo bash -c "cat > $tuning_file" << EOF
# worker MPM
# StartServers: initial number of server processes to start
# MinSpareThreads: minimum number of worker threads which are kept spare
# MaxSpareThreads: maximum number of worker threads which are kept spare
# ThreadLimit: ThreadsPerChild can be changed to this maximum value during a
# graceful restart. ThreadLimit can only be changed by stopping
# and starting Apache.
# ThreadsPerChild: constant number of worker threads in each server process
# MaxClients: maximum number of simultaneous client connections
# MaxRequestsPerChild: maximum number of requests a server process serves
#
# The apache defaults are too conservative if we want reliable tempest
# testing. Bump these values up from ~400 max clients to 1024 max clients.
<IfModule mpm_worker_module>
# Note that the next three conf values must be changed together.
# MaxClients = ServerLimit * ThreadsPerChild
ServerLimit 32
ThreadsPerChild 32
MaxClients 1024
StartServers 3
MinSpareThreads 96
MaxSpareThreads 192
ThreadLimit 64
MaxRequestsPerChild 0
</IfModule>
<IfModule mpm_event_module>
# Note that the next three conf values must be changed together.
# MaxClients = ServerLimit * ThreadsPerChild
ServerLimit 32
ThreadsPerChild 32
MaxClients 1024
StartServers 3
MinSpareThreads 96
MaxSpareThreads 192
ThreadLimit 64
MaxRequestsPerChild 0
</IfModule>
EOF
restart_apache_server
fi
}
# Starts the TLS proxy for the given IP/ports
# start_tls_proxy front-host front-port back-host back-port
function start_tls_proxy {
local b_service="$1-tls-proxy"
local f_host=$2
local f_port=$3
local b_host=$4
local b_port=$5
tune_apache_connections
local config_file
config_file=$(apache_site_config_for $b_service)
local listen_string
# Default apache configs on ubuntu and centos listen on 80 and 443
# newer apache seems fine with duplicate listen directive but older
# apache does not so special case 80 and 443.
if [[ "$f_port" == "80" ]] || [[ "$f_port" == "443" ]]; then
listen_string=""
elif [[ "$f_host" == '*' ]] ; then
listen_string="Listen $f_port"
else
listen_string="Listen $f_host:$f_port"
fi
sudo bash -c "cat >$config_file" << EOF
$listen_string
<VirtualHost $f_host:$f_port>
SSLEngine On
SSLCertificateFile $DEVSTACK_CERT
<Location />
ProxyPass http://$b_host:$b_port/ retry=5 nocanon
ProxyPassReverse http://$b_host:$b_port/
</Location>
ErrorLog $APACHE_LOG_DIR/tls-proxy_error.log
ErrorLogFormat "[%{u}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] [frontend\ %A] %M% ,\ referer\ %{Referer}i"
LogLevel info
CustomLog $APACHE_LOG_DIR/tls-proxy_access.log common
LogFormat "%v %h %l %u %t \"%r\" %>s %b"
</VirtualHost>
EOF
for mod in ssl proxy proxy_http; do
enable_apache_mod $mod
done
enable_apache_site $b_service
# Only a reload is required to pull in new vhosts
# Note that a restart reliably fails on centos7 and trusty
# because apache can't open port 80 because the old apache
# still has it open. Using reload fixes trusty but centos7
# still doesn't work.
reload_apache_server
}
# Follow TLS proxy
function follow_tls_proxy {
sudo touch /var/log/$APACHE_NAME/tls-proxy_error.log
tail_log tls-error /var/log/$APACHE_NAME/tls-proxy_error.log
sudo touch /var/log/$APACHE_NAME/tls-proxy_access.log
tail_log tls-proxy /var/log/$APACHE_NAME/tls-proxy_access.log
}
# Cleanup Functions
# =================
# Stops the apache service. This should be done only after all services
# using tls configuration are down.
function stop_tls_proxy {
stop_apache_server
}
# Clean up the CA files
# cleanup_CA
function cleanup_CA {
if is_fedora; then
sudo rm -f /usr/share/pki/ca-trust-source/anchors/devstack-chain.pem
sudo update-ca-trust
elif is_ubuntu; then
sudo rm -f /usr/local/share/ca-certificates/devstack-int.crt
sudo rm -f /usr/local/share/ca-certificates/devstack-root.crt
sudo update-ca-certificates
fi
rm -rf "$INT_CA_DIR" "$ROOT_CA_DIR" "$DEVSTACK_CERT"
}
# Tell emacs to use shell-script-mode
## Local variables:
## mode: shell-script
## End:
View Git Blame Copy Permalink