diff --git a/lib/cinder b/lib/cinder
index 387fc1ac2c..07f82a1580 100644
--- a/lib/cinder
+++ b/lib/cinder
@@ -51,6 +51,7 @@ else
 fi
 
 CINDER_STATE_PATH=${CINDER_STATE_PATH:=$DATA_DIR/cinder}
+CINDER_AUTH_CACHE_DIR=${CINDER_AUTH_CACHE_DIR:-/var/cache/cinder}
 
 CINDER_CONF_DIR=/etc/cinder
 CINDER_CONF=$CINDER_CONF_DIR/cinder.conf
@@ -224,8 +225,9 @@ function configure_cinder {
     inicomment $CINDER_API_PASTE_INI filter:authtoken admin_tenant_name
     inicomment $CINDER_API_PASTE_INI filter:authtoken admin_user
     inicomment $CINDER_API_PASTE_INI filter:authtoken admin_password
+    inicomment $CINDER_API_PASTE_INI filter:authtoken signing_dir
 
-    configure_auth_token_middleware $CINDER_CONF cinder
+    configure_auth_token_middleware $CINDER_CONF cinder $CINDER_AUTH_CACHE_DIR
 
     iniset $CINDER_CONF DEFAULT auth_strategy keystone
     iniset $CINDER_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
@@ -383,6 +385,13 @@ function create_cinder_accounts {
     fi
 }
 
+# create_cinder_cache_dir() - Part of the init_cinder() process
+function create_cinder_cache_dir {
+    # Create cache dir
+    sudo install -d -o $STACK_USER $CINDER_AUTH_CACHE_DIR
+    rm -f $CINDER_AUTH_CACHE_DIR/*
+}
+
 # init_cinder() - Initialize database and volume group
 function init_cinder {
     if is_service_enabled $DATABASE_BACKENDS; then
@@ -411,6 +420,7 @@ function init_cinder {
     fi
 
     mkdir -p $CINDER_STATE_PATH/volumes
+    create_cinder_cache_dir
 }
 
 # install_cinder() - Collect source and prepare
diff --git a/lib/glance b/lib/glance
index 8241c5f9d7..74734c7516 100644
--- a/lib/glance
+++ b/lib/glance
@@ -44,6 +44,7 @@ fi
 GLANCE_CACHE_DIR=${GLANCE_CACHE_DIR:=$DATA_DIR/glance/cache}
 GLANCE_IMAGE_DIR=${GLANCE_IMAGE_DIR:=$DATA_DIR/glance/images}
 GLANCE_LOCK_DIR=${GLANCE_LOCK_DIR:=$DATA_DIR/glance/locks}
+GLANCE_AUTH_CACHE_DIR=${GLANCE_AUTH_CACHE_DIR:-/var/cache/glance}
 
 GLANCE_CONF_DIR=${GLANCE_CONF_DIR:-/etc/glance}
 GLANCE_METADEF_DIR=$GLANCE_CONF_DIR/metadefs
@@ -97,7 +98,7 @@ function is_glance_enabled {
 function cleanup_glance {
     # kill instances (nova)
     # delete image files (glance)
-    sudo rm -rf $GLANCE_CACHE_DIR $GLANCE_IMAGE_DIR
+    sudo rm -rf $GLANCE_CACHE_DIR $GLANCE_IMAGE_DIR $GLANCE_AUTH_CACHE_DIR
 }
 
 # configure_glance() - Set config files, create data dirs, etc
@@ -114,7 +115,7 @@ function configure_glance {
     iniset $GLANCE_REGISTRY_CONF database connection $dburl
     iniset $GLANCE_REGISTRY_CONF DEFAULT use_syslog $SYSLOG
     iniset $GLANCE_REGISTRY_CONF paste_deploy flavor keystone
-    configure_auth_token_middleware $GLANCE_REGISTRY_CONF glance
+    configure_auth_token_middleware $GLANCE_REGISTRY_CONF glance $GLANCE_AUTH_CACHE_DIR/registry
     iniset $GLANCE_REGISTRY_CONF oslo_messaging_notifications driver messagingv2
     iniset_rpc_backend glance $GLANCE_REGISTRY_CONF
     iniset $GLANCE_REGISTRY_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT"
@@ -126,7 +127,7 @@ function configure_glance {
     iniset $GLANCE_API_CONF DEFAULT image_cache_dir $GLANCE_CACHE_DIR/
     iniset $GLANCE_API_CONF DEFAULT lock_path $GLANCE_LOCK_DIR
     iniset $GLANCE_API_CONF paste_deploy flavor keystone+cachemanagement
-    configure_auth_token_middleware $GLANCE_API_CONF glance
+    configure_auth_token_middleware $GLANCE_API_CONF glance $GLANCE_AUTH_CACHE_DIR/api
     iniset $GLANCE_API_CONF oslo_messaging_notifications driver messagingv2
     iniset_rpc_backend glance $GLANCE_API_CONF
     if [ "$VIRT_DRIVER" = 'xenserver' ]; then
@@ -278,6 +279,13 @@ function create_glance_accounts {
     fi
 }
 
+# create_glance_cache_dir() - Part of the init_glance() process
+function create_glance_cache_dir {
+    # Create cache dir
+    sudo install -d -o $STACK_USER $GLANCE_AUTH_CACHE_DIR/api $GLANCE_AUTH_CACHE_DIR/registry $GLANCE_AUTH_CACHE_DIR/search $GLANCE_AUTH_CACHE_DIR/artifact
+    rm -f $GLANCE_AUTH_CACHE_DIR/api/* $GLANCE_AUTH_CACHE_DIR/registry/* $GLANCE_AUTH_CACHE_DIR/search/* $GLANCE_AUTH_CACHE_DIR/artifact/*
+}
+
 # init_glance() - Initialize databases, etc.
 function init_glance {
     # Delete existing images
@@ -298,6 +306,8 @@ function init_glance {
     # Load metadata definitions
     $GLANCE_BIN_DIR/glance-manage --config-file $GLANCE_CONF_DIR/glance-api.conf db_load_metadefs
     time_stop "dbsync"
+
+    create_glance_cache_dir
 }
 
 # install_glanceclient() - Collect source and prepare
diff --git a/lib/keystone b/lib/keystone
index 7bd887ca2d..714f089cca 100644
--- a/lib/keystone
+++ b/lib/keystone
@@ -429,7 +429,7 @@ function create_service_user {
 
 # Configure the service to use the auth token middleware.
 #
-# configure_auth_token_middleware conf_file admin_user [section]
+# configure_auth_token_middleware conf_file admin_user signing_dir [section]
 #
 # section defaults to keystone_authtoken, which is where auth_token looks in
 # the .conf file. If the paste config file is used (api-paste.ini) then
@@ -437,7 +437,8 @@ function create_service_user {
 function configure_auth_token_middleware {
     local conf_file=$1
     local admin_user=$2
-    local section=${3:-keystone_authtoken}
+    local signing_dir=$3
+    local section=${4:-keystone_authtoken}
 
     iniset $conf_file $section auth_type password
     iniset $conf_file $section auth_url $KEYSTONE_SERVICE_URI
@@ -448,6 +449,7 @@ function configure_auth_token_middleware {
     iniset $conf_file $section project_domain_name "$SERVICE_DOMAIN_NAME"
 
     iniset $conf_file $section cafile $SSL_BUNDLE_FILE
+    iniset $conf_file $section signing_dir $signing_dir
     iniset $conf_file $section memcached_servers localhost:11211
 }
 
diff --git a/lib/neutron b/lib/neutron
index 359f19820d..21c8d4c735 100644
--- a/lib/neutron
+++ b/lib/neutron
@@ -30,6 +30,7 @@ GITDIR["python-neutronclient"]=$DEST/python-neutronclient
 
 NEUTRON_AGENT=${NEUTRON_AGENT:-openvswitch}
 NEUTRON_DIR=$DEST/neutron
+NEUTRON_AUTH_CACHE_DIR=${NEUTRON_AUTH_CACHE_DIR:-/var/cache/neutron}
 
 NEUTRON_BIN_DIR=$(get_python_exec_prefix)
 NEUTRON_DHCP_BINARY="neutron-dhcp-agent"
@@ -43,6 +44,7 @@ NEUTRON_L3_CONF=$NEUTRON_CONF_DIR/l3_agent.ini
 NEUTRON_AGENT_CONF=$NEUTRON_CONF_DIR/
 
 NEUTRON_STATE_PATH=${NEUTRON_STATE_PATH:=$DATA_DIR/neutron}
+NEUTRON_AUTH_CACHE_DIR=${NEUTRON_AUTH_CACHE_DIR:-/var/cache/neutron}
 
 # By default, use the ML2 plugin
 NEUTRON_CORE_PLUGIN=${NEUTRON_CORE_PLUGIN:-ml2}
@@ -173,8 +175,8 @@ function configure_neutron_new {
         iniset $NEUTRON_CONF DEFAULT allow_overlapping_ips True
 
         iniset $NEUTRON_CONF DEFAULT auth_strategy $NEUTRON_AUTH_STRATEGY
-        configure_auth_token_middleware $NEUTRON_CONF neutron keystone_authtoken
-        configure_auth_token_middleware $NEUTRON_CONF nova nova
+        configure_auth_token_middleware $NEUTRON_CONF neutron $NEUTRON_AUTH_CACHE_DIR keystone_authtoken
+        configure_auth_token_middleware $NEUTRON_CONF nova $NEUTRON_AUTH_CACHE_DIR nova
 
         # Configure VXLAN
         # TODO(sc68cal) not hardcode?
@@ -248,7 +250,7 @@ function configure_neutron_new {
 
         # TODO(dtroyer): remove the v2.0 hard code below
         iniset $NEUTRON_META_CONF DEFAULT auth_url $KEYSTONE_SERVICE_URI
-        configure_auth_token_middleware $NEUTRON_META_CONF neutron DEFAULT
+        configure_auth_token_middleware $NEUTRON_META_CONF neutron $NEUTRON_AUTH_CACHE_DIR DEFAULT
     fi
 
     # Format logging
@@ -335,6 +337,13 @@ function create_neutron_accounts_new {
     fi
 }
 
+# create_neutron_cache_dir() - Part of the init_neutron() process
+function create_neutron_cache_dir {
+    # Create cache dir
+    sudo install -d -o $STACK_USER $NEUTRON_AUTH_CACHE_DIR
+    rm -f $NEUTRON_AUTH_CACHE_DIR/*
+}
+
 # init_neutron() - Initialize databases, etc.
 function init_neutron_new {
 
@@ -344,6 +353,8 @@ function init_neutron_new {
     # Run Neutron db migrations
     $NEUTRON_BIN_DIR/neutron-db-manage upgrade heads
     time_stop "dbsync"
+
+    create_neutron_cache_dir
 }
 
 # install_neutron() - Collect source and prepare
diff --git a/lib/neutron-legacy b/lib/neutron-legacy
index a0e79bc684..0ccb17c084 100644
--- a/lib/neutron-legacy
+++ b/lib/neutron-legacy
@@ -73,6 +73,7 @@ GITDIR["python-neutronclient"]=$DEST/python-neutronclient
 
 NEUTRON_DIR=$DEST/neutron
 NEUTRON_FWAAS_DIR=$DEST/neutron-fwaas
+NEUTRON_AUTH_CACHE_DIR=${NEUTRON_AUTH_CACHE_DIR:-/var/cache/neutron}
 
 # Support entry points installation of console scripts
 if [[ -d $NEUTRON_DIR/bin/neutron-server ]]; then
@@ -814,7 +815,7 @@ function _configure_neutron_service {
     iniset $NEUTRON_CONF DEFAULT notify_nova_on_port_status_changes $Q_NOTIFY_NOVA_PORT_STATUS_CHANGES
     iniset $NEUTRON_CONF DEFAULT notify_nova_on_port_data_changes $Q_NOTIFY_NOVA_PORT_DATA_CHANGES
 
-    configure_auth_token_middleware $NEUTRON_CONF nova nova
+    configure_auth_token_middleware $NEUTRON_CONF nova $NEUTRON_AUTH_CACHE_DIR nova
 
     # Configure plugin
     neutron_plugin_configure_service
@@ -905,7 +906,8 @@ function _neutron_setup_keystone {
     local conf_file=$1
     local section=$2
 
-    configure_auth_token_middleware $conf_file $Q_ADMIN_USERNAME $section
+    create_neutron_cache_dir
+    configure_auth_token_middleware $conf_file $Q_ADMIN_USERNAME $NEUTRON_AUTH_CACHE_DIR $section
 }
 
 function _neutron_setup_interface_driver {
diff --git a/lib/nova b/lib/nova
index dcf2a1c376..1112f29bd3 100644
--- a/lib/nova
+++ b/lib/nova
@@ -46,6 +46,7 @@ fi
 NOVA_STATE_PATH=${NOVA_STATE_PATH:=$DATA_DIR/nova}
 # INSTANCES_PATH is the previous name for this
 NOVA_INSTANCES_PATH=${NOVA_INSTANCES_PATH:=${INSTANCES_PATH:=$NOVA_STATE_PATH/instances}}
+NOVA_AUTH_CACHE_DIR=${NOVA_AUTH_CACHE_DIR:-/var/cache/nova}
 
 NOVA_CONF_DIR=/etc/nova
 NOVA_CONF=$NOVA_CONF_DIR/nova.conf
@@ -239,7 +240,7 @@ function cleanup_nova {
         sudo rm -rf $NOVA_INSTANCES_PATH/*
     fi
 
-    sudo rm -rf $NOVA_STATE_PATH
+    sudo rm -rf $NOVA_STATE_PATH $NOVA_AUTH_CACHE_DIR
 
     # NOTE(dtroyer): This really should be called from here but due to the way
     #                nova abuses the _cleanup() function we're moving it
@@ -463,7 +464,7 @@ function create_nova_conf {
             iniset $NOVA_CONF DEFAULT osapi_compute_link_prefix $NOVA_SERVICE_PROTOCOL://$NOVA_SERVICE_HOST:$NOVA_SERVICE_PORT
         fi
 
-        configure_auth_token_middleware $NOVA_CONF nova
+        configure_auth_token_middleware $NOVA_CONF nova $NOVA_AUTH_CACHE_DIR
     fi
 
     if is_service_enabled cinder; then
@@ -657,6 +658,13 @@ function init_nova_cells {
     fi
 }
 
+# create_nova_cache_dir() - Part of the init_nova() process
+function create_nova_cache_dir {
+    # Create cache dir
+    sudo install -d -o $STACK_USER $NOVA_AUTH_CACHE_DIR
+    rm -f $NOVA_AUTH_CACHE_DIR/*
+}
+
 function create_nova_conf_nova_network {
     local public_interface=${PUBLIC_INTERFACE:-$PUBLIC_INTERFACE_DEFAULT}
     iniset $NOVA_CONF DEFAULT network_manager "nova.network.manager.$NETWORK_MANAGER"
@@ -714,6 +722,7 @@ function init_nova {
         done
     fi
 
+    create_nova_cache_dir
     create_nova_keys_dir
 
     if [[ "$NOVA_BACKEND" == "LVM" ]]; then
diff --git a/lib/swift b/lib/swift
index ab014de909..1601e2b1f8 100644
--- a/lib/swift
+++ b/lib/swift
@@ -48,6 +48,7 @@ fi
 
 
 SWIFT_DIR=$DEST/swift
+SWIFT_AUTH_CACHE_DIR=${SWIFT_AUTH_CACHE_DIR:-/var/cache/swift}
 SWIFT_APACHE_WSGI_DIR=${SWIFT_APACHE_WSGI_DIR:-/var/www/swift}
 SWIFT3_DIR=$DEST/swift3
 
@@ -449,7 +450,7 @@ function configure_swift {
     iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:authtoken log_name swift
 
     iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:authtoken paste.filter_factory keystonemiddleware.auth_token:filter_factory
-    configure_auth_token_middleware $SWIFT_CONFIG_PROXY_SERVER swift filter:authtoken
+    configure_auth_token_middleware $SWIFT_CONFIG_PROXY_SERVER swift $SWIFT_AUTH_CACHE_DIR filter:authtoken
     iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:authtoken delay_auth_decision 1
     iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:authtoken cache swift.cache
     iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:authtoken include_service_catalog False
@@ -744,6 +745,10 @@ function init_swift {
         swift-ring-builder container.builder rebalance 42
         swift-ring-builder account.builder rebalance 42
     } && popd >/dev/null
+
+    # Create cache dir
+    sudo install -d -o ${STACK_USER} $SWIFT_AUTH_CACHE_DIR
+    rm -f $SWIFT_AUTH_CACHE_DIR/*
 }
 
 function install_swift {