openstack/devstack
Code Issues Proposed changes
0572d73f85
devstack/lib/tls
Dan Smith 64d68679d9 Improve API log parsing 
Two runs of the same job on the same patch can yield quite different
numbers for API calls if we just count the raw calls. Many of these
are tempest polling for resources, which on a slow worker can require
many more calls than a fast one.

Tempest seems to not change its User-Agent string, but the client
libraries do. So, if we ignore the regular "python-urllib" agent
calls, we get a much more stable count of service-to-service API
calls in the performance report.

Note that we were also logging in a different (less-rich) format for
the tls-proxy.log file, which hampers our ability to parse that
data in the same format. This switches it to "combined" which is used
by the access.log and contains more useful information, like the
user-agent, among other things.

Change-Id: I8889c2e53f85c41150e1245dcbe2a79bac702aad
2022-05-12 07:55:30 -07:00

615 lines
18 KiB
Bash
Raw Blame History

#!/bin/bash
#
# lib/tls
# Functions to control the configuration and operation of the TLS proxy service
# !! source _before_ any services that use ``SERVICE_HOST``
#
# Dependencies:
#
# - ``functions`` file
# - ``DEST``, ``DATA_DIR`` must be defined
# - ``HOST_IP``, ``SERVICE_HOST``
# - ``KEYSTONE_TOKEN_FORMAT`` must be defined
# Entry points:
#
# - configure_CA
# - init_CA
# - configure_proxy
# - start_tls_proxy
# - stop_tls_proxy
# - cleanup_CA
# - make_root_CA
# - make_int_CA
# - make_cert ca-dir cert-name "common-name" ["alt-name" ...]
# - start_tls_proxy HOST_IP 5000 localhost 5000
# - ensure_certificates
# - is_ssl_enabled_service
# - enable_mod_ssl
# Defaults
# --------
if is_service_enabled tls-proxy; then
# TODO(dtroyer): revisit this below after the search for HOST_IP has been done
TLS_IP=${TLS_IP:-$(ipv6_unquote $SERVICE_HOST)}
fi
DEVSTACK_HOSTNAME=$(hostname -f)
DEVSTACK_CERT_NAME=devstack-cert
DEVSTACK_CERT=$DATA_DIR/$DEVSTACK_CERT_NAME.pem
# CA configuration
ROOT_CA_DIR=${ROOT_CA_DIR:-$DATA_DIR/CA/root-ca}
INT_CA_DIR=${INT_CA_DIR:-$DATA_DIR/CA/int-ca}
ORG_NAME="OpenStack"
ORG_UNIT_NAME="DevStack"
# Stud configuration
STUD_PROTO="--tls"
STUD_CIPHERS='TLSv1+HIGH:!DES:!aNULL:!eNULL:@STRENGTH'
# CA Functions
# ============
# There may be more than one, get specific
OPENSSL=${OPENSSL:-/usr/bin/openssl}
# Do primary CA configuration
function configure_CA {
# build common config file
# Verify ``TLS_IP`` is good
if [[ -n "$SERVICE_HOST" && "$(ipv6_unquote $SERVICE_HOST)" != "$TLS_IP" ]]; then
# auto-discover has changed the IP
TLS_IP=$(ipv6_unquote $SERVICE_HOST)
fi
}
# Creates a new CA directory structure
# create_CA_base ca-dir
function create_CA_base {
local ca_dir=$1
if [[ -d $ca_dir ]]; then
# Bail out it exists
return 0
fi
local i
for i in certs crl newcerts private; do
mkdir -p $ca_dir/$i
done
chmod 710 $ca_dir/private
echo "01" >$ca_dir/serial
cp /dev/null $ca_dir/index.txt
}
# Create a new CA configuration file
# create_CA_config ca-dir common-name
function create_CA_config {
local ca_dir=$1
local common_name=$2
echo "
[ ca ]
default_ca = CA_default
[ CA_default ]
dir = $ca_dir
policy = policy_match
database = \$dir/index.txt
serial = \$dir/serial
certs = \$dir/certs
crl_dir = \$dir/crl
new_certs_dir = \$dir/newcerts
certificate = \$dir/cacert.pem
private_key = \$dir/private/cacert.key
RANDFILE = \$dir/private/.rand
default_md = sha256
[ req ]
default_bits = 2048
default_md = sha256
prompt = no
distinguished_name = ca_distinguished_name
x509_extensions = ca_extensions
[ ca_distinguished_name ]
organizationName = $ORG_NAME
organizationalUnitName = $ORG_UNIT_NAME Certificate Authority
commonName = $common_name
[ policy_match ]
countryName = optional
stateOrProvinceName = optional
organizationName = match
organizationalUnitName = optional
commonName = supplied
[ ca_extensions ]
basicConstraints = critical,CA:true
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always, issuer
keyUsage = cRLSign, keyCertSign
" >$ca_dir/ca.conf
}
# Create a new signing configuration file
# create_signing_config ca-dir
function create_signing_config {
local ca_dir=$1
echo "
[ ca ]
default_ca = CA_default
[ CA_default ]
dir = $ca_dir
policy = policy_match
database = \$dir/index.txt
serial = \$dir/serial
certs = \$dir/certs
crl_dir = \$dir/crl
new_certs_dir = \$dir/newcerts
certificate = \$dir/cacert.pem
private_key = \$dir/private/cacert.key
RANDFILE = \$dir/private/.rand
default_md = default
[ req ]
default_bits = 1024
default_md = sha256
prompt = no
distinguished_name = req_distinguished_name
x509_extensions = req_extensions
[ req_distinguished_name ]
organizationName = $ORG_NAME
organizationalUnitName = $ORG_UNIT_NAME Server Farm
[ policy_match ]
countryName = optional
stateOrProvinceName = optional
organizationName = match
organizationalUnitName = optional
commonName = supplied
[ req_extensions ]
basicConstraints = CA:false
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always, issuer
keyUsage = digitalSignature, keyEncipherment, keyAgreement
extendedKeyUsage = serverAuth, clientAuth
subjectAltName = \$ENV::SUBJECT_ALT_NAME
" >$ca_dir/signing.conf
}
# Create root and intermediate CAs
# init_CA
function init_CA {
# Ensure CAs are built
make_root_CA $ROOT_CA_DIR
make_int_CA $INT_CA_DIR $ROOT_CA_DIR
# Create the CA bundle
cat $ROOT_CA_DIR/cacert.pem $INT_CA_DIR/cacert.pem >>$INT_CA_DIR/ca-chain.pem
cat $INT_CA_DIR/ca-chain.pem >> $SSL_BUNDLE_FILE
if is_fedora; then
sudo cp $INT_CA_DIR/ca-chain.pem /usr/share/pki/ca-trust-source/anchors/devstack-chain.pem
sudo update-ca-trust
elif is_suse; then
sudo cp $INT_CA_DIR/ca-chain.pem /usr/share/pki/trust/anchors/devstack-chain.pem
sudo update-ca-certificates
elif is_ubuntu; then
sudo cp $INT_CA_DIR/ca-chain.pem /usr/local/share/ca-certificates/devstack-int.crt
sudo cp $ROOT_CA_DIR/cacert.pem /usr/local/share/ca-certificates/devstack-root.crt
sudo update-ca-certificates
fi
}
# Create an initial server cert
# init_cert
function init_cert {
if [[ ! -r $DEVSTACK_CERT ]]; then
if [[ -n "$TLS_IP" ]]; then
TLS_IP="IP:$TLS_IP"
if [[ -n "$HOST_IPV6" ]]; then
TLS_IP="$TLS_IP,IP:$HOST_IPV6"
fi
fi
make_cert $INT_CA_DIR $DEVSTACK_CERT_NAME $DEVSTACK_HOSTNAME "$TLS_IP"
# Create a cert bundle
cat $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/cacert.pem >$DEVSTACK_CERT
fi
}
# make_cert creates and signs a new certificate with the given commonName and CA
# make_cert ca-dir cert-name "common-name" ["alt-name" ...]
function make_cert {
local ca_dir=$1
local cert_name=$2
local common_name=$3
local alt_names=$4
if [ "$common_name" != "$SERVICE_HOST" ]; then
if is_ipv4_address "$SERVICE_HOST" ; then
if [[ -z "$alt_names" ]]; then
alt_names="IP:$SERVICE_HOST"
else
alt_names="$alt_names,IP:$SERVICE_HOST"
fi
fi
fi
# Only generate the certificate if it doesn't exist yet on the disk
if [ ! -r "$ca_dir/$cert_name.crt" ]; then
# Generate a signing request
$OPENSSL req \
-sha256 \
-newkey rsa \
-nodes \
-keyout $ca_dir/private/$cert_name.key \
-out $ca_dir/$cert_name.csr \
-subj "/O=${ORG_NAME}/OU=${ORG_UNIT_NAME} Servers/CN=${common_name}"
if [[ -z "$alt_names" ]]; then
alt_names="DNS:${common_name}"
else
alt_names="DNS:${common_name},${alt_names}"
fi
# Sign the request valid for 1 year
SUBJECT_ALT_NAME="$alt_names" \
$OPENSSL ca -config $ca_dir/signing.conf \
-extensions req_extensions \
-days 365 \
-notext \
-in $ca_dir/$cert_name.csr \
-out $ca_dir/$cert_name.crt \
-subj "/O=${ORG_NAME}/OU=${ORG_UNIT_NAME} Servers/CN=${common_name}" \
-batch
fi
}
# Make an intermediate CA to sign everything else
# make_int_CA ca-dir signing-ca-dir
function make_int_CA {
local ca_dir=$1
local signing_ca_dir=$2
# Create the root CA
create_CA_base $ca_dir
create_CA_config $ca_dir 'Intermediate CA'
create_signing_config $ca_dir
if [ ! -r "$ca_dir/cacert.pem" ]; then
# Create a signing certificate request
$OPENSSL req -config $ca_dir/ca.conf \
-sha256 \
-newkey rsa \
-nodes \
-keyout $ca_dir/private/cacert.key \
-out $ca_dir/cacert.csr \
-outform PEM
# Sign the intermediate request valid for 1 year
$OPENSSL ca -config $signing_ca_dir/ca.conf \
-extensions ca_extensions \
-days 365 \
-notext \
-in $ca_dir/cacert.csr \
-out $ca_dir/cacert.pem \
-batch
fi
}
# Make a root CA to sign other CAs
# make_root_CA ca-dir
function make_root_CA {
local ca_dir=$1
# Create the root CA
create_CA_base $ca_dir
create_CA_config $ca_dir 'Root CA'
if [ ! -r "$ca_dir/cacert.pem" ]; then
# Create a self-signed certificate valid for 5 years
$OPENSSL req -config $ca_dir/ca.conf \
-x509 \
-nodes \
-newkey rsa \
-days 21360 \
-keyout $ca_dir/private/cacert.key \
-out $ca_dir/cacert.pem \
-outform PEM
fi
}
# Deploy the service cert & key to a service specific
# location
function deploy_int_cert {
local cert_target_file=$1
local key_target_file=$2
sudo cp "$INT_CA_DIR/$DEVSTACK_CERT_NAME.crt" "$cert_target_file"
sudo cp "$INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key" "$key_target_file"
}
# Deploy the intermediate CA cert bundle file to a service
# specific location
function deploy_int_CA {
local ca_target_file=$1
sudo cp "$INT_CA_DIR/ca-chain.pem" "$ca_target_file"
}
# If a non-system python-requests is installed then it will use the
# built-in CA certificate store rather than the distro-specific
# CA certificate store. Detect this and symlink to the correct
# one. If the value for the CA is not rooted in /etc then we know
# we need to change it.
function fix_system_ca_bundle_path {
if is_service_enabled tls-proxy; then
local capath
capath=$(python3 -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass')
if [[ ! $capath == "" && ! $capath =~ ^/etc/.* && ! -L $capath ]]; then
if is_fedora; then
sudo rm -f $capath
sudo ln -s /etc/pki/tls/certs/ca-bundle.crt $capath
elif is_ubuntu; then
sudo rm -f $capath
sudo ln -s /etc/ssl/certs/ca-certificates.crt $capath
elif is_suse; then
sudo rm -f $capath
sudo ln -s /etc/ssl/ca-bundle.pem $capath
else
echo "Don't know how to set the CA bundle, expect the install to fail."
fi
fi
fi
}
# Only for compatibility, return if the tls-proxy is enabled
function is_ssl_enabled_service {
return is_service_enabled tls-proxy
}
# Certificate Input Configuration
# ===============================
# Ensure that the certificates for a service are in place. This function does
# not check that a service is SSL enabled, this should already have been
# completed.
#
# The function expects to find a certificate, key and CA certificate in the
# variables ``{service}_SSL_CERT``, ``{service}_SSL_KEY`` and ``{service}_SSL_CA``. For
# example for keystone this would be ``KEYSTONE_SSL_CERT``, ``KEYSTONE_SSL_KEY`` and
# ``KEYSTONE_SSL_CA``.
#
# If it does not find these certificates then the DevStack-issued server
# certificate, key and CA certificate will be associated with the service.
#
# If only some of the variables are provided then the function will quit.
function ensure_certificates {
local service=$1
local cert_var="${service}_SSL_CERT"
local key_var="${service}_SSL_KEY"
local ca_var="${service}_SSL_CA"
local cert=${!cert_var}
local key=${!key_var}
local ca=${!ca_var}
if [[ -z "$cert" && -z "$key" && -z "$ca" ]]; then
local cert="$INT_CA_DIR/$DEVSTACK_CERT_NAME.crt"
local key="$INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key"
local ca="$INT_CA_DIR/ca-chain.pem"
eval ${service}_SSL_CERT=\$cert
eval ${service}_SSL_KEY=\$key
eval ${service}_SSL_CA=\$ca
return # the CA certificate is already in the bundle
elif [[ -z "$cert" || -z "$key" || -z "$ca" ]]; then
die $LINENO "Missing either the ${cert_var} ${key_var} or ${ca_var}" \
"variable to enable SSL for ${service}"
fi
cat $ca >> $SSL_BUNDLE_FILE
}
# Enable the mod_ssl plugin in Apache
function enable_mod_ssl {
echo "Enabling mod_ssl"
if is_ubuntu; then
sudo a2enmod ssl
elif is_suse; then
sudo a2enmod ssl
sudo a2enflag SSL
elif is_fedora; then
# Fedora enables mod_ssl by default
:
fi
if ! sudo `which httpd || which apache2ctl` -M | grep -w -q ssl_module; then
die $LINENO "mod_ssl is not enabled in apache2/httpd, please check for it manually and run stack.sh again"
fi
}
# Proxy Functions
# ===============
function tune_apache_connections {
local tuning_file=$APACHE_SETTINGS_DIR/connection-tuning.conf
if ! [ -f $tuning_file ] ; then
sudo bash -c "cat > $tuning_file" << EOF
# worker MPM
# StartServers: initial number of server processes to start
# MinSpareThreads: minimum number of worker threads which are kept spare
# MaxSpareThreads: maximum number of worker threads which are kept spare
# ThreadLimit: ThreadsPerChild can be changed to this maximum value during a
# graceful restart. ThreadLimit can only be changed by stopping
# and starting Apache.
# ThreadsPerChild: constant number of worker threads in each server process
# MaxClients: maximum number of simultaneous client connections
# MaxRequestsPerChild: maximum number of requests a server process serves
#
# We want to be memory thrifty so tune down apache to allow 256 total
# connections. This should still be plenty for a dev env yet lighter than
# apache defaults.
<IfModule mpm_worker_module>
# Note that the next three conf values must be changed together.
# MaxClients = ServerLimit * ThreadsPerChild
ServerLimit 8
ThreadsPerChild 32
MaxClients 256
StartServers 2
MinSpareThreads 32
MaxSpareThreads 96
ThreadLimit 64
MaxRequestsPerChild 0
</IfModule>
<IfModule mpm_event_module>
# Note that the next three conf values must be changed together.
# MaxClients = ServerLimit * ThreadsPerChild
ServerLimit 8
ThreadsPerChild 32
MaxClients 256
StartServers 2
MinSpareThreads 32
MaxSpareThreads 96
ThreadLimit 64
MaxRequestsPerChild 0
</IfModule>
EOF
restart_apache_server
fi
}
# Starts the TLS proxy for the given IP/ports
# start_tls_proxy service-name front-host front-port back-host back-port
function start_tls_proxy {
local b_service="$1-tls-proxy"
local f_host=$2
local f_port=$3
local b_host=$4
local b_port=$5
# 8190 is the default apache size.
local f_header_size=${6:-8190}
tune_apache_connections
local config_file
config_file=$(apache_site_config_for $b_service)
local listen_string
# Default apache configs on ubuntu and centos listen on 80 and 443
# newer apache seems fine with duplicate listen directive but older
# apache does not so special case 80 and 443.
if [[ "$f_port" == "80" ]] || [[ "$f_port" == "443" ]]; then
listen_string=""
elif [[ "$f_host" == '*' ]] ; then
listen_string="Listen $f_port"
else
listen_string="Listen $f_host:$f_port"
fi
sudo bash -c "cat >$config_file" << EOF
$listen_string
<VirtualHost $f_host:$f_port>
SSLEngine On
SSLCertificateFile $DEVSTACK_CERT
# Disable KeepAlive to fix bug #1630664 a.k.a the
# ('Connection aborted.', BadStatusLine("''",)) error
KeepAlive Off
# This increase in allowed request header sizes is required
# for swift functional testing to work with tls enabled. It is 2 bytes
# larger than the apache default of 8190.
LimitRequestFieldSize $f_header_size
RequestHeader set X-Forwarded-Proto "https"
# Avoid races (at the cost of performance) to re-use a pooled connection
# where the connection is closed (bug 1807518).
SetEnv proxy-initial-not-pooled
<Location />
ProxyPass http://$b_host:$b_port/ retry=0 nocanon
ProxyPassReverse http://$b_host:$b_port/
</Location>
ErrorLog $APACHE_LOG_DIR/tls-proxy_error.log
ErrorLogFormat "%{cu}t [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] [frontend\ %A] %M% ,\ referer\ %{Referer}i"
LogLevel info
CustomLog $APACHE_LOG_DIR/tls-proxy_access.log combined
</VirtualHost>
EOF
if is_suse ; then
sudo a2enflag SSL
fi
for mod in headers ssl proxy proxy_http; do
enable_apache_mod $mod
done
enable_apache_site $b_service
restart_apache_server
}
# Cleanup Functions
# =================
# Stops the apache service. This should be done only after all services
# using tls configuration are down.
function stop_tls_proxy {
stop_apache_server
# NOTE(jh): Removing all tls-proxy configs is a bit of a hack, but
# necessary so that we can restart after an unstack. A better
# solution would be to ensure that each service calling
# start_tls_proxy will call stop_tls_proxy with the same
# parameters on shutdown so we can use the disable_apache_site
# function and remove individual files there.
if is_ubuntu; then
sudo rm -f /etc/apache2/sites-enabled/*-tls-proxy.conf
else
for i in $APACHE_CONF_DIR/*-tls-proxy.conf; do
sudo mv $i $i.disabled
done
fi
}
# Clean up the CA files
# cleanup_CA
function cleanup_CA {
if is_fedora; then
sudo rm -f /usr/share/pki/ca-trust-source/anchors/devstack-chain.pem
sudo update-ca-trust
elif is_ubuntu; then
sudo rm -f /usr/local/share/ca-certificates/devstack-int.crt
sudo rm -f /usr/local/share/ca-certificates/devstack-root.crt
sudo update-ca-certificates
fi
rm -rf "$INT_CA_DIR" "$ROOT_CA_DIR" "$DEVSTACK_CERT"
}
# Tell emacs to use shell-script-mode
## Local variables:
## mode: shell-script
## End:
View Git Blame Copy Permalink