From 90fd9152a4f4b9c018a28df5cabecb0607cd0af7 Mon Sep 17 00:00:00 2001 From: Michal Arbet Date: Fri, 13 Aug 2021 22:32:53 +0200 Subject: [PATCH] Use Docker healthchecks for keystone-fernet container This change enables the use of Docker healthchecks for keystone-fernet container. It checks if "key 0" has right permissions, and if rsync is able to distribute keys to other keystones. Implements: blueprint container-health-check Change-Id: I17bea723d4109e869cd05d211f6f8e4653f46e17 --- ansible/roles/keystone/defaults/main.yml | 14 ++++++++ ansible/roles/keystone/tasks/config.yml | 1 + .../templates/fernet-healthcheck.sh.j2 | 6 ++++ .../keystone/templates/fernet-node-sync.sh.j2 | 34 ++++++++++++------- .../keystone/templates/fernet-push.sh.j2 | 16 +++++++++ .../templates/keystone-fernet.json.j2 | 6 ++++ ...-for-keystone-fernet-a63033e2b95ecb2f.yaml | 6 ++++ 7 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 ansible/roles/keystone/templates/fernet-healthcheck.sh.j2 create mode 100644 releasenotes/notes/implement-docker-healthchecks-for-keystone-fernet-a63033e2b95ecb2f.yaml diff --git a/ansible/roles/keystone/defaults/main.yml b/ansible/roles/keystone/defaults/main.yml index e07bda1304..cb4aa206b3 100644 --- a/ansible/roles/keystone/defaults/main.yml +++ b/ansible/roles/keystone/defaults/main.yml @@ -59,6 +59,7 @@ keystone_services: - "kolla_logs:/var/log/kolla/" - "keystone_fernet_tokens:/etc/keystone/fernet-keys" dimensions: "{{ keystone_fernet_dimensions }}" + healthcheck: "{{ keystone_fernet_healthcheck }}" #################### # Database @@ -123,6 +124,19 @@ keystone_ssh_healthcheck: test: "{% if keystone_ssh_enable_healthchecks | bool %}{{ keystone_ssh_healthcheck_test }}{% else %}NONE{% endif %}" timeout: "{{ keystone_ssh_healthcheck_timeout }}" +keystone_fernet_enable_healthchecks: "{{ enable_container_healthchecks }}" +keystone_fernet_healthcheck_interval: "{{ default_container_healthcheck_interval }}" +keystone_fernet_healthcheck_retries: "{{ default_container_healthcheck_retries }}" +keystone_fernet_healthcheck_start_period: "{{ default_container_healthcheck_start_period }}" +keystone_fernet_healthcheck_test: ["CMD-SHELL", "/usr/bin/fernet-healthcheck.sh"] +keystone_fernet_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}" +keystone_fernet_healthcheck: + interval: "{{ keystone_fernet_healthcheck_interval }}" + retries: "{{ keystone_fernet_healthcheck_retries }}" + start_period: "{{ keystone_fernet_healthcheck_start_period }}" + test: "{% if keystone_fernet_enable_healthchecks | bool %}{{ keystone_fernet_healthcheck_test }}{% else %}NONE{% endif %}" + timeout: "{{ keystone_fernet_healthcheck_timeout }}" + keystone_default_volumes: - "{{ node_config_directory }}/keystone/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" diff --git a/ansible/roles/keystone/tasks/config.yml b/ansible/roles/keystone/tasks/config.yml index bec1350a34..ffccd79023 100644 --- a/ansible/roles/keystone/tasks/config.yml +++ b/ansible/roles/keystone/tasks/config.yml @@ -223,6 +223,7 @@ - { src: "fernet-rotate.sh.j2", dest: "fernet-rotate.sh" } - { src: "fernet-node-sync.sh.j2", dest: "fernet-node-sync.sh" } - { src: "fernet-push.sh.j2", dest: "fernet-push.sh" } + - { src: "fernet-healthcheck.sh.j2", dest: "fernet-healthcheck.sh" } - { src: "id_rsa", dest: "id_rsa" } - { src: "ssh_config.j2", dest: "ssh_config" } when: diff --git a/ansible/roles/keystone/templates/fernet-healthcheck.sh.j2 b/ansible/roles/keystone/templates/fernet-healthcheck.sh.j2 new file mode 100644 index 0000000000..29c71bd322 --- /dev/null +++ b/ansible/roles/keystone/templates/fernet-healthcheck.sh.j2 @@ -0,0 +1,6 @@ +#!/bin/bash + +set -o errexit +set -o pipefail + +(/usr/bin/fernet-node-sync.sh --check && /usr/bin/fernet-push.sh --check) || exit 1 diff --git a/ansible/roles/keystone/templates/fernet-node-sync.sh.j2 b/ansible/roles/keystone/templates/fernet-node-sync.sh.j2 index 4293a807d8..27a1b497f9 100644 --- a/ansible/roles/keystone/templates/fernet-node-sync.sh.j2 +++ b/ansible/roles/keystone/templates/fernet-node-sync.sh.j2 @@ -1,19 +1,29 @@ -#!/bin/bash +!/bin/bash set -o errexit set -o pipefail -# Ensure tokens are populated, check for 0 key which should always exist -n=0 -while [ ! -f /etc/keystone/fernet-keys/0 ]; do - if [ $n -lt 10 ]; then - n=$(( n + 1 )) - echo "ERROR: Fernet tokens have not been populated, rechecking in 1 minute" - echo "DEBUG: /etc/keystone/fernet-keys contents:" - ls -l /etc/keystone/fernet-keys/ - sleep 60 +if [ ! -z "$1" ] && [ "$1" == "--check" ]; then + if [ -f /etc/keystone/fernet-keys/0 ]; then + if [[ $(stat -c %U:%G /etc/keystone/fernet-keys/0) != "keystone:keystone" ]]; then + exit 1 + fi else - echo "CRITICAL: Waited for 10 minutes - failing" exit 1 fi -done +else + # Ensure tokens are populated, check for 0 key which should always exist + n=0 + while [ ! -f /etc/keystone/fernet-keys/0 ]; do + if [ $n -lt 10 ]; then + n=$(( n + 1 )) + echo "ERROR: Fernet tokens have not been populated, rechecking in 1 minute" + echo "DEBUG: /etc/keystone/fernet-keys contents:" + ls -l /etc/keystone/fernet-keys/ + sleep 60 + else + echo "CRITICAL: Waited for 10 minutes - failing" + exit 1 + fi + done +fi diff --git a/ansible/roles/keystone/templates/fernet-push.sh.j2 b/ansible/roles/keystone/templates/fernet-push.sh.j2 index 5aeda018fc..2b0ee6fac2 100644 --- a/ansible/roles/keystone/templates/fernet-push.sh.j2 +++ b/ansible/roles/keystone/templates/fernet-push.sh.j2 @@ -3,8 +3,24 @@ set -o errexit set -o pipefail +if [ ! -z "$1" ] && [ "$1" == "--check" ]; then +{% if groups['keystone'] | length > 1 %} +{% for host in groups['keystone'] %} +{% if inventory_hostname != host %} +/usr/bin/rsync --dry-run -az -e 'ssh -i /var/lib/keystone/.ssh/id_rsa -p {{ hostvars[host]['keystone_ssh_port'] }} -F /var/lib/keystone/.ssh/config' --delete /etc/keystone/fernet-keys/ keystone@{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:/etc/keystone/fernet-keys +{% endif %} +{% endfor %} +{% else %} +echo "No additional keystone-server where fernet keys could be rsynced." +{% endif %} +else +{% if groups['keystone'] | length > 1 %} {% for host in groups['keystone'] %} {% if inventory_hostname != host %} /usr/bin/rsync -az -e 'ssh -i /var/lib/keystone/.ssh/id_rsa -p {{ hostvars[host]['keystone_ssh_port'] }} -F /var/lib/keystone/.ssh/config' --delete /etc/keystone/fernet-keys/ keystone@{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:/etc/keystone/fernet-keys {% endif %} {% endfor %} +{% else %} +echo "No additional keystone-server where fernet keys could be rsynced." +{% endif %} +fi diff --git a/ansible/roles/keystone/templates/keystone-fernet.json.j2 b/ansible/roles/keystone/templates/keystone-fernet.json.j2 index 2486a59e84..208e0dd922 100644 --- a/ansible/roles/keystone/templates/keystone-fernet.json.j2 +++ b/ansible/roles/keystone/templates/keystone-fernet.json.j2 @@ -49,6 +49,12 @@ "dest": "/etc/keystone/{{ keystone_policy_file }}", "owner": "keystone", "perm": "0600" + }{% endif %}{% if keystone_fernet_enable_healthchecks | bool %}, + { + "source": "{{ container_config_directory }}/fernet-healthcheck.sh", + "dest": "/usr/bin/fernet-healthcheck.sh", + "owner": "root", + "perm": "0755" }{% endif %} ], "permissions": [ diff --git a/releasenotes/notes/implement-docker-healthchecks-for-keystone-fernet-a63033e2b95ecb2f.yaml b/releasenotes/notes/implement-docker-healthchecks-for-keystone-fernet-a63033e2b95ecb2f.yaml new file mode 100644 index 0000000000..26b0b26337 --- /dev/null +++ b/releasenotes/notes/implement-docker-healthchecks-for-keystone-fernet-a63033e2b95ecb2f.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Implements container healthchecks for keystone-fernet container. + See `blueprint + `__