From ba8c27f554682e1f2720fad1bff5cfa1b35017f2 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Thu, 22 Oct 2020 09:18:32 +0100
Subject: [PATCH] Fix keystone-startup.sh - remove Fernet key age check
Currently we check the age of the primary Fernet key on Keystone
startup, and fail if it is older than the rotation interval. While this
may seem sensible, there are various reasons why the key may be older
than this:
* if the rotation interval is not a factor of the number of seconds in a
week, the rotation schedule will be lumpy, with the last rotation
being up to twice the nominal rotation interval
* if a keystone host is unavailable at its scheduled rotation time,
rotation will not happen. This may happen multiple times
We could do several things to avoid this issue:
1. remove the check on the age of the key
2. multiply the rotation interval by some factor to determine the
allowed key age
This change goes for the more simple option 1. It also cleans up some
terminology in the keystone-startup.sh script.
Closes-Bug: #1895723
Change-Id: I2c35f59ae9449cb1646e402e0a9f28ad61f918a8
---
.../keystone/templates/keystone-startup.sh.j2 | 25 ++++++-------------
...fix-keystone-startup-66c5aa11a464a562.yaml | 8 ++++++
2 files changed, 15 insertions(+), 18 deletions(-)
create mode 100644 releasenotes/notes/fix-keystone-startup-66c5aa11a464a562.yaml
diff --git a/ansible/roles/keystone/templates/keystone-startup.sh.j2 b/ansible/roles/keystone/templates/keystone-startup.sh.j2
index 2304df1ab2..2ea21744cd 100644
--- a/ansible/roles/keystone/templates/keystone-startup.sh.j2
+++ b/ansible/roles/keystone/templates/keystone-startup.sh.j2
@@ -4,32 +4,21 @@
set -o errexit
set -o pipefail
-TOKEN_DIR="/etc/keystone/fernet-keys"
+FERNET_KEY_DIR="/etc/keystone/fernet-keys"
-# Ensure tokens are populated, check for 0 (staging) key
+# Ensure Fernet keys are populated, check for 0 (staging) key
n=0
-while [ ! -f "${TOKEN_DIR}/0" ]; do
+while [ ! -f "${FERNET_KEY_DIR}/0" ]; do
if [ $n -lt 36 ]; then
n=$(( n + 1 ))
- echo "ERROR: Fernet tokens have not been populated, rechecking in 5 seconds"
- echo "DEBUG: ${TOKEN_DIR} contents:"
- ls -l ${TOKEN_DIR}
+ echo "ERROR: Fernet keys have not been populated, rechecking in 5 seconds"
+ echo "DEBUG: ${FERNET_KEY_DIR} contents:"
+ ls -l ${FERNET_KEY_DIR}
sleep 5
else
- echo "CRITICAL: Waited for 10 minutes - failing"
+ echo "CRITICAL: Waited for 3 minutes - failing"
exit 1
fi
done
-# Ensure tokens are not stale
-# Get primary token (file with highest number)
-TOKEN_PRIMARY=$(ls -1 ${TOKEN_DIR} | sort -hr | head -n 1)
-# Check it's age in seconds
-TOKEN_AGE=$(($(date +%s) - $(date +%s -r "${TOKEN_DIR}/${TOKEN_PRIMARY}")))
-# Compare if it's older than fernet_key_rotation_interval and fail if it's stale
-if [ "${TOKEN_AGE}" -gt "{{ fernet_key_rotation_interval }}" ]; then
- echo "ERROR: Primary token ${TOKEN_PRIMARY} is stale."
- exit 1
-fi
-
exec /usr/sbin/{{ keystone_cmd }} $@
diff --git a/releasenotes/notes/fix-keystone-startup-66c5aa11a464a562.yaml b/releasenotes/notes/fix-keystone-startup-66c5aa11a464a562.yaml
new file mode 100644
index 0000000000..37c425331a
--- /dev/null
+++ b/releasenotes/notes/fix-keystone-startup-66c5aa11a464a562.yaml
@@ -0,0 +1,8 @@
+---
+fixes:
+ - |
+ Fixes an issue with Keystone startup when Fernet key rotation does not
+ occur within the configured interval. This may happen due to one of the
+ Keystone hosts being down at the scheduled time of rotation, or due to
+ uneven intervals between cron jobs. `LP#1895723
+ <https://bugs.launchpad.net/kolla-ansible/+bug/1895723>`__