c3b0b9034e
This will cause apache to no longer wait forever for a connection pool member to become available before returning 503 to the client. This may help us determine if some of the timeouts we see when talking to the services come from an overloaded apache. Change-Id: Ibc19fc9a53e2330f9aca45f5a10a59c576cb22e6
395 lines
14 KiB
Bash
395 lines
14 KiB
Bash
#!/bin/bash
|
|
#
|
|
# lib/apache
|
|
# Functions to control configuration and operation of apache web server
|
|
|
|
# Dependencies:
|
|
#
|
|
# - ``functions`` file
|
|
# - ``STACK_USER`` must be defined
|
|
#
|
|
# lib/apache exports the following functions:
|
|
#
|
|
# - install_apache_wsgi
|
|
# - apache_site_config_for
|
|
# - enable_apache_site
|
|
# - disable_apache_site
|
|
# - start_apache_server
|
|
# - stop_apache_server
|
|
# - restart_apache_server
|
|
|
|
# Save trace setting
|
|
_XTRACE_LIB_APACHE=$(set +o | grep xtrace)
|
|
set +o xtrace
|
|
|
|
# Allow overriding the default Apache user and group, default to
|
|
# current user and his default group.
|
|
APACHE_USER=${APACHE_USER:-$STACK_USER}
|
|
APACHE_GROUP=${APACHE_GROUP:-$(id -gn $APACHE_USER)}
|
|
|
|
APACHE_LOCAL_HOST=$SERVICE_LOCAL_HOST
|
|
if [[ "$SERVICE_IP_VERSION" == 6 ]]; then
|
|
APACHE_LOCAL_HOST=[$APACHE_LOCAL_HOST]
|
|
fi
|
|
|
|
|
|
# Set up apache name and configuration directory
|
|
# Note that APACHE_CONF_DIR is really more accurately apache's vhost
|
|
# configuration dir but we can't just change this because public interfaces.
|
|
if is_ubuntu; then
|
|
APACHE_NAME=apache2
|
|
APACHE_CONF_DIR=${APACHE_CONF_DIR:-/etc/$APACHE_NAME/sites-available}
|
|
APACHE_SETTINGS_DIR=${APACHE_SETTINGS_DIR:-/etc/$APACHE_NAME/conf-enabled}
|
|
elif is_fedora; then
|
|
APACHE_NAME=httpd
|
|
APACHE_CONF_DIR=${APACHE_CONF_DIR:-/etc/$APACHE_NAME/conf.d}
|
|
APACHE_SETTINGS_DIR=${APACHE_SETTINGS_DIR:-/etc/$APACHE_NAME/conf.d}
|
|
fi
|
|
APACHE_LOG_DIR="/var/log/${APACHE_NAME}"
|
|
|
|
# Functions
|
|
# ---------
|
|
|
|
# Enable apache mod and restart apache if it isn't already enabled.
|
|
function enable_apache_mod {
|
|
local mod=$1
|
|
# Apache installation, because we mark it NOPRIME
|
|
if is_ubuntu; then
|
|
# Skip mod_version as it is not a valid mod to enable
|
|
# on debuntu, instead it is built in.
|
|
if [[ "$mod" != "version" ]] && ! a2query -m $mod ; then
|
|
sudo a2enmod $mod
|
|
restart_apache_server
|
|
fi
|
|
elif is_fedora; then
|
|
# pass
|
|
true
|
|
else
|
|
exit_distro_not_supported "apache enable mod"
|
|
fi
|
|
}
|
|
|
|
# NOTE(sdague): Install uwsgi including apache module, we need to get
|
|
# to 2.0.6+ to get a working mod_proxy_uwsgi. We can probably build a
|
|
# check for that and do it differently for different platforms.
|
|
function install_apache_uwsgi {
|
|
local apxs="apxs2"
|
|
if is_fedora; then
|
|
apxs="apxs"
|
|
fi
|
|
|
|
if is_ubuntu; then
|
|
local pkg_list="uwsgi uwsgi-plugin-python3 libapache2-mod-proxy-uwsgi"
|
|
install_package ${pkg_list}
|
|
# NOTE(ianw) 2022-02-03 : Fedora 35 needs to skip this and fall
|
|
# into the install-from-source because the upstream packages
|
|
# didn't fix Python 3.10 compatibility before release. Should be
|
|
# fixed in uwsgi 4.9.0; can remove this when packages available
|
|
# or we drop this release
|
|
elif is_fedora && ! is_openeuler && ! [[ $DISTRO =~ f36 ]]; then
|
|
# Note httpd comes with mod_proxy_uwsgi and it is loaded by
|
|
# default; the mod_proxy_uwsgi package actually conflicts now.
|
|
# See:
|
|
# https://bugzilla.redhat.com/show_bug.cgi?id=1574335
|
|
#
|
|
# Thus there is nothing else to do after this install
|
|
install_package uwsgi \
|
|
uwsgi-plugin-python3
|
|
else
|
|
# Compile uwsgi from source.
|
|
local dir
|
|
dir=$(mktemp -d)
|
|
pushd $dir
|
|
pip_install uwsgi
|
|
pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt
|
|
local uwsgi
|
|
uwsgi=$(ls uwsgi*)
|
|
tar xvf $uwsgi
|
|
cd uwsgi*/apache2
|
|
sudo $apxs -i -c mod_proxy_uwsgi.c
|
|
popd
|
|
# delete the temp directory
|
|
sudo rm -rf $dir
|
|
fi
|
|
|
|
if is_ubuntu; then
|
|
# we've got to enable proxy and proxy_uwsgi for this to work
|
|
sudo a2enmod proxy
|
|
sudo a2enmod proxy_uwsgi
|
|
elif is_fedora; then
|
|
# redhat is missing a nice way to turn on/off modules
|
|
echo "LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so" \
|
|
| sudo tee /etc/httpd/conf.modules.d/02-proxy-uwsgi.conf
|
|
fi
|
|
restart_apache_server
|
|
}
|
|
|
|
# install_apache_wsgi() - Install Apache server and wsgi module
|
|
function install_apache_wsgi {
|
|
# Apache installation, because we mark it NOPRIME
|
|
if is_ubuntu; then
|
|
# Install apache2, which is NOPRIME'd
|
|
install_package apache2
|
|
if is_package_installed libapache2-mod-wsgi; then
|
|
uninstall_package libapache2-mod-wsgi
|
|
fi
|
|
install_package libapache2-mod-wsgi-py3
|
|
elif is_fedora; then
|
|
sudo rm -f /etc/httpd/conf.d/000-*
|
|
install_package httpd python3-mod_wsgi
|
|
# rpm distros dont enable httpd by default so enable it to support reboots.
|
|
sudo systemctl enable httpd
|
|
# For consistency with Ubuntu, switch to the worker mpm, as
|
|
# the default is event
|
|
sudo sed -i '/mod_mpm_prefork.so/s/^/#/g' /etc/httpd/conf.modules.d/00-mpm.conf
|
|
sudo sed -i '/mod_mpm_event.so/s/^/#/g' /etc/httpd/conf.modules.d/00-mpm.conf
|
|
sudo sed -i '/mod_mpm_worker.so/s/^#//g' /etc/httpd/conf.modules.d/00-mpm.conf
|
|
else
|
|
exit_distro_not_supported "apache wsgi installation"
|
|
fi
|
|
# WSGI isn't enabled by default, enable it
|
|
enable_apache_mod wsgi
|
|
}
|
|
|
|
# apache_site_config_for() - The filename of the site's configuration file.
|
|
# This function uses the global variables APACHE_NAME and APACHE_CONF_DIR.
|
|
#
|
|
# On Ubuntu 14.04+, the site configuration file must have a .conf suffix for a2ensite and a2dissite to
|
|
# recognise it. a2ensite and a2dissite ignore the .conf suffix used as parameter. The default sites'
|
|
# files are 000-default.conf and default-ssl.conf.
|
|
#
|
|
# On Fedora, any file in /etc/httpd/conf.d/ whose name ends with .conf is enabled.
|
|
#
|
|
# On RHEL and CentOS, things should hopefully work as in Fedora.
|
|
#
|
|
# The table below summarizes what should happen on each distribution:
|
|
# +----------------------+--------------------+--------------------------+--------------------------+
|
|
# | Distribution | File name | Site enabling command | Site disabling command |
|
|
# +----------------------+--------------------+--------------------------+--------------------------+
|
|
# | Ubuntu 14.04 | site.conf | a2ensite site | a2dissite site |
|
|
# | Fedora, RHEL, CentOS | site.conf.disabled | mv site.conf{.disabled,} | mv site.conf{,.disabled} |
|
|
# +----------------------+--------------------+--------------------------+--------------------------+
|
|
function apache_site_config_for {
|
|
local site=$@
|
|
if is_ubuntu; then
|
|
# Ubuntu 14.04 - Apache 2.4
|
|
echo $APACHE_CONF_DIR/${site}.conf
|
|
elif is_fedora; then
|
|
# fedora conf.d is only imported if it ends with .conf so this is approx the same
|
|
local enabled_site_file="$APACHE_CONF_DIR/${site}.conf"
|
|
if [ -f $enabled_site_file ]; then
|
|
echo ${enabled_site_file}
|
|
else
|
|
echo ${enabled_site_file}.disabled
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# enable_apache_site() - Enable a particular apache site
|
|
function enable_apache_site {
|
|
local site=$@
|
|
# Many of our sites use mod version. Just enable it.
|
|
enable_apache_mod version
|
|
if is_ubuntu; then
|
|
sudo a2ensite ${site}
|
|
elif is_fedora; then
|
|
local enabled_site_file="$APACHE_CONF_DIR/${site}.conf"
|
|
# Do nothing if site already enabled or no site config exists
|
|
if [[ -f ${enabled_site_file}.disabled ]] && [[ ! -f ${enabled_site_file} ]]; then
|
|
sudo mv ${enabled_site_file}.disabled ${enabled_site_file}
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# disable_apache_site() - Disable a particular apache site
|
|
function disable_apache_site {
|
|
local site=$@
|
|
if is_ubuntu; then
|
|
sudo a2dissite ${site} || true
|
|
elif is_fedora; then
|
|
local enabled_site_file="$APACHE_CONF_DIR/${site}.conf"
|
|
# Do nothing if no site config exists
|
|
if [[ -f ${enabled_site_file} ]]; then
|
|
sudo mv ${enabled_site_file} ${enabled_site_file}.disabled
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# start_apache_server() - Start running apache server
|
|
function start_apache_server {
|
|
start_service $APACHE_NAME
|
|
}
|
|
|
|
# stop_apache_server() - Stop running apache server
|
|
function stop_apache_server {
|
|
if [ -n "$APACHE_NAME" ]; then
|
|
stop_service $APACHE_NAME
|
|
else
|
|
exit_distro_not_supported "apache configuration"
|
|
fi
|
|
}
|
|
|
|
# restart_apache_server
|
|
function restart_apache_server {
|
|
# Apache can be slow to stop, doing an explicit stop, sleep, start helps
|
|
# to mitigate issues where apache will claim a port it's listening on is
|
|
# still in use and fail to start.
|
|
restart_service $APACHE_NAME
|
|
}
|
|
|
|
function write_uwsgi_config {
|
|
local file=$1
|
|
local wsgi=$2
|
|
local url=$3
|
|
local http=$4
|
|
local name=""
|
|
name=$(basename $wsgi)
|
|
|
|
# create a home for the sockets; note don't use /tmp -- apache has
|
|
# a private view of it on some platforms.
|
|
local socket_dir='/var/run/uwsgi'
|
|
|
|
# /var/run will be empty on ubuntu after reboot, so we can use systemd-temptiles
|
|
# to automatically create $socket_dir.
|
|
sudo mkdir -p /etc/tmpfiles.d/
|
|
echo "d $socket_dir 0755 $STACK_USER root" | sudo tee /etc/tmpfiles.d/uwsgi.conf
|
|
sudo systemd-tmpfiles --create /etc/tmpfiles.d/uwsgi.conf
|
|
|
|
local socket="$socket_dir/${name}.socket"
|
|
|
|
# always cleanup given that we are using iniset here
|
|
rm -rf $file
|
|
iniset "$file" uwsgi wsgi-file "$wsgi"
|
|
iniset "$file" uwsgi processes $API_WORKERS
|
|
# This is running standalone
|
|
iniset "$file" uwsgi master true
|
|
# Set die-on-term & exit-on-reload so that uwsgi shuts down
|
|
iniset "$file" uwsgi die-on-term true
|
|
iniset "$file" uwsgi exit-on-reload false
|
|
# Set worker-reload-mercy so that worker will not exit till the time
|
|
# configured after graceful shutdown
|
|
iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
|
|
iniset "$file" uwsgi enable-threads true
|
|
iniset "$file" uwsgi plugins http,python3
|
|
# uwsgi recommends this to prevent thundering herd on accept.
|
|
iniset "$file" uwsgi thunder-lock true
|
|
# Set hook to trigger graceful shutdown on SIGTERM
|
|
iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
|
|
# Override the default size for headers from the 4k default.
|
|
iniset "$file" uwsgi buffer-size 65535
|
|
# Make sure the client doesn't try to re-use the connection.
|
|
iniset "$file" uwsgi add-header "Connection: close"
|
|
# This ensures that file descriptors aren't shared between processes.
|
|
iniset "$file" uwsgi lazy-apps true
|
|
|
|
# If we said bind directly to http, then do that and don't start the apache proxy
|
|
if [[ -n "$http" ]]; then
|
|
iniset "$file" uwsgi http $http
|
|
else
|
|
local apache_conf=""
|
|
apache_conf=$(apache_site_config_for $name)
|
|
iniset "$file" uwsgi socket "$socket"
|
|
iniset "$file" uwsgi chmod-socket 666
|
|
echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}\" retry=0 acquire=1 " | sudo tee -a $apache_conf
|
|
enable_apache_site $name
|
|
restart_apache_server
|
|
fi
|
|
}
|
|
|
|
# For services using chunked encoding, the only services known to use this
|
|
# currently are Glance and Swift, we need to use an http proxy instead of
|
|
# mod_proxy_uwsgi because the chunked encoding gets dropped. See:
|
|
# https://github.com/unbit/uwsgi/issues/1540 You can workaround this on python2
|
|
# but that involves having apache buffer the request before sending it to
|
|
# uwsgi.
|
|
function write_local_uwsgi_http_config {
|
|
local file=$1
|
|
local wsgi=$2
|
|
local url=$3
|
|
name=$(basename $wsgi)
|
|
|
|
# create a home for the sockets; note don't use /tmp -- apache has
|
|
# a private view of it on some platforms.
|
|
|
|
# always cleanup given that we are using iniset here
|
|
rm -rf $file
|
|
iniset "$file" uwsgi wsgi-file "$wsgi"
|
|
port=$(get_random_port)
|
|
iniset "$file" uwsgi http-socket "$APACHE_LOCAL_HOST:$port"
|
|
iniset "$file" uwsgi processes $API_WORKERS
|
|
# This is running standalone
|
|
iniset "$file" uwsgi master true
|
|
# Set die-on-term & exit-on-reload so that uwsgi shuts down
|
|
iniset "$file" uwsgi die-on-term true
|
|
iniset "$file" uwsgi exit-on-reload false
|
|
iniset "$file" uwsgi enable-threads true
|
|
iniset "$file" uwsgi plugins http,python3
|
|
# uwsgi recommends this to prevent thundering herd on accept.
|
|
iniset "$file" uwsgi thunder-lock true
|
|
# Set hook to trigger graceful shutdown on SIGTERM
|
|
iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
|
|
# Set worker-reload-mercy so that worker will not exit till the time
|
|
# configured after graceful shutdown
|
|
iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
|
|
# Override the default size for headers from the 4k default.
|
|
iniset "$file" uwsgi buffer-size 65535
|
|
# Make sure the client doesn't try to re-use the connection.
|
|
iniset "$file" uwsgi add-header "Connection: close"
|
|
# This ensures that file descriptors aren't shared between processes.
|
|
iniset "$file" uwsgi lazy-apps true
|
|
iniset "$file" uwsgi chmod-socket 666
|
|
iniset "$file" uwsgi http-raw-body true
|
|
iniset "$file" uwsgi http-chunked-input true
|
|
iniset "$file" uwsgi http-auto-chunked true
|
|
iniset "$file" uwsgi http-keepalive false
|
|
# Increase socket timeout for slow chunked uploads
|
|
iniset "$file" uwsgi socket-timeout 30
|
|
|
|
enable_apache_mod proxy
|
|
enable_apache_mod proxy_http
|
|
local apache_conf=""
|
|
apache_conf=$(apache_site_config_for $name)
|
|
echo "KeepAlive Off" | sudo tee $apache_conf
|
|
echo "SetEnv proxy-sendchunked 1" | sudo tee -a $apache_conf
|
|
echo "ProxyPass \"${url}\" \"http://$APACHE_LOCAL_HOST:$port\" retry=0 acquire=1 " | sudo tee -a $apache_conf
|
|
enable_apache_site $name
|
|
restart_apache_server
|
|
}
|
|
|
|
# Write a straight-through proxy for a service that runs locally and just needs
|
|
# to be reachable via the main http proxy at $loc
|
|
function write_local_proxy_http_config {
|
|
local name=$1
|
|
local url=$2
|
|
local loc=$3
|
|
local apache_conf
|
|
apache_conf=$(apache_site_config_for $name)
|
|
|
|
enable_apache_mod proxy
|
|
enable_apache_mod proxy_http
|
|
|
|
echo "KeepAlive Off" | sudo tee $apache_conf
|
|
echo "SetEnv proxy-sendchunked 1" | sudo tee -a $apache_conf
|
|
echo "ProxyPass \"${loc}\" \"$url\" retry=0 acquire=1 " | sudo tee -a $apache_conf
|
|
enable_apache_site $name
|
|
restart_apache_server
|
|
}
|
|
|
|
function remove_uwsgi_config {
|
|
local file=$1
|
|
local wsgi=$2
|
|
local name=""
|
|
name=$(basename $wsgi)
|
|
|
|
rm -rf $file
|
|
disable_apache_site $name
|
|
}
|
|
|
|
# Restore xtrace
|
|
$_XTRACE_LIB_APACHE
|
|
|
|
# Tell emacs to use shell-script-mode
|
|
## Local variables:
|
|
## mode: shell-script
|
|
## End:
|