Add full support for fernet

This addresses the ansible aspects of fernet key bootstrapping as
well as distributed key rotation.

- Bootstrapping is handled in the same way as keystone bootstrap.
- A new keystone-fernet and keystone-ssh container is created to allow
  the nodes to communicate with each other (taken from nova-ssh).
- The keystone-fernet is a keystone container with crontab installed.
  This will handle key rotations through keystone-manage and trigger
  an rsync to push new tokens to other nodes.
- Key rotation is setup to be balanced across the keystone nodes using
  a round-robbin style. This ensures that any node failures will not
  stop the keys from rotating. This is configured by a desired token
  expiration time which then determines the cron scheduling for each
  node as well as the number of fernet tokens in rotation.
- Ability for recovered node to resync with the cluster. When a node
  starts it will run sanity checks to ensure that its fernet tokens
  are not stale. If they are it will rsync with other nodes to ensure
  its tokens are up to date.

The Docker component is implemented in:
  https://review.openstack.org/#/c/349366

Change-Id: I15052c25a1d1149d364236f10ced2e2346119738
Implements: blueprint keystone-fernet-token

ansible/group_vars/all.yml

@@ -131,6 +131,7 @@ haproxy_stats_port: "1984"
 
 keystone_public_port: "5000"
 keystone_admin_port: "35357"
+keystone_ssh_port: "8023"
 
 glance_api_port: "9292"
 glance_registry_port: "9191"
@@ -282,6 +283,13 @@ keystone_internal_url: "{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ k
 keystone_public_url: "{{ public_protocol }}://{{ kolla_external_fqdn }}:{{ keystone_public_port }}/v3"
 
 
+#####################################
+# Keystone - Identity Service options
+#####################################
+keystone_token_provider: "uuid"
+fernet_token_expiry: 86400
+
+
 #######################
 # Glance options
 #######################

ansible/roles/keystone/defaults/main.yml

@@ -9,6 +9,13 @@ keystone_database_user: "keystone"
 keystone_database_address: "{{ kolla_internal_fqdn }}:{{ database_port }}"
 
 
+####################
+# Fernet
+####################
+keystone_username: "keystone"
+keystone_groupname: "keystone"
+
+
 ####################
 # Docker
 ####################
@@ -16,6 +23,14 @@ keystone_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker
 keystone_tag: "{{ openstack_release }}"
 keystone_image_full: "{{ keystone_image }}:{{ keystone_tag }}"
 
+keystone_fernet_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-keystone-fernet"
+keystone_fernet_tag: "{{ openstack_release }}"
+keystone_fernet_image_full: "{{ keystone_fernet_image }}:{{ keystone_fernet_tag }}"
+
+keystone_ssh_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-keystone-ssh"
+keystone_ssh_tag: "{{ openstack_release }}"
+keystone_ssh_image_full: "{{ keystone_ssh_image }}:{{ keystone_ssh_tag }}"
+
 
 ####################
 # OpenStack

ansible/roles/keystone/files/fernet_rotate_cron_generator.py

@@ -0,0 +1,107 @@
+#!/usr/bin/python
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This module creates a list of cron intervals for a node in a group of nodes
+# to ensure each node runs a cron in round robbin style.
+
+from __future__ import print_function
+import argparse
+import json
+import sys
+
+MINUTE_SPAN = 1
+HOUR_SPAN = 60
+DAY_SPAN = 24 * HOUR_SPAN
+WEEK_SPAN = 7 * DAY_SPAN
+
+
+def json_exit(msg=None, failed=False, changed=False):
+    if type(msg) is not dict:
+        msg = {'msg': str(msg)}
+    msg.update({'failed': failed, 'changed': changed})
+    print(json.dumps(msg))
+    sys.exit()
+
+
+def generate(host_index, total_hosts, total_rotation_mins):
+    min = '*'
+    hour = '*'
+    day = '*'
+    crons = []
+
+    if host_index >= total_hosts:
+        return crons
+
+    rotation_frequency = total_rotation_mins // total_hosts
+    cron_min = rotation_frequency * host_index
+
+    # Build crons for a week period
+    if total_rotation_mins == WEEK_SPAN:
+        day = cron_min // DAY_SPAN
+        hour = (cron_min % DAY_SPAN) // HOUR_SPAN
+        min = cron_min % HOUR_SPAN
+        crons.append({'min': min, 'hour': hour, 'day': day})
+
+    # Build crons for a day period
+    elif total_rotation_mins == DAY_SPAN:
+        hour = cron_min // HOUR_SPAN
+        min = cron_min % HOUR_SPAN
+        crons.append({'min': min, 'hour': hour, 'day': day})
+
+    # Build crons for multiple of an hour
+    elif total_rotation_mins % HOUR_SPAN == 0:
+        for multiple in range(1, DAY_SPAN // total_rotation_mins + 1):
+            time = cron_min
+            if multiple > 1:
+                time += total_rotation_mins * (multiple - 1)
+
+            hour = time // HOUR_SPAN
+            min = time % HOUR_SPAN
+            crons.append({'min': min, 'hour': hour, 'day': day})
+
+    # Build crons for multiple of a minute
+    elif total_rotation_mins % MINUTE_SPAN == 0:
+        for multiple in range(1, HOUR_SPAN // total_rotation_mins + 1):
+            time = cron_min
+            if multiple > 1:
+                time += total_rotation_mins * (multiple - 1)
+
+            min = time // MINUTE_SPAN
+            crons.append({'min': min, 'hour': hour, 'day': day})
+
+    return crons
+
+
+def main():
+    parser = argparse.ArgumentParser(description='''Creates a list of cron
+        intervals for a node in a group of nodes to ensure each node runs
+        a cron in round robbin style.''')
+    parser.add_argument('-t', '--time',
+                        help='Time in seconds for a token rotation cycle',
+                        required=True,
+                        type=int)
+    parser.add_argument('-i', '--index',
+                        help='Index of host starting from 0',
+                        required=True,
+                        type=int)
+    parser.add_argument('-n', '--number',
+                        help='Number of tokens that should exist',
+                        required=True,
+                        type=int)
+    args = parser.parse_args()
+    json_exit({'cron_jobs': generate(args.index, args.number, args.time)})
+
+
+if __name__ == "__main__":
+    main()

ansible/roles/keystone/tasks/config.yml

@@ -14,6 +14,8 @@
     recurse: yes
   with_items:
     - "keystone"
+    - "keystone-fernet"
+    - "keystone-ssh"
 
 - name: Creating Keystone Domain directory
   file:
@@ -30,6 +32,8 @@
     dest: "{{ node_config_directory }}/{{ item }}/config.json"
   with_items:
     - "keystone"
+    - "keystone-fernet"
+    - "keystone-ssh"
 
 - name: Copying over keystone.conf
   merge_configs:
@@ -45,6 +49,8 @@
     dest: "{{ node_config_directory }}/{{ item }}/keystone.conf"
   with_items:
     - "keystone"
+    - "keystone-fernet"
+    - "keystone-ssh"
 
 - name: Copying Keystone Domain specific settings
   copy:
@@ -68,3 +74,34 @@
     - "{{ node_custom_config }}/keystone/{{ inventory_hostname }}/wsgi-keystone.conf"
     - "{{ node_custom_config }}/keystone/wsgi-keystone.conf"
     - "wsgi-keystone.conf.j2"
+
+- name: Generate the required cron jobs for the node
+  local_action: "command python {{ role_path }}/files/fernet_rotate_cron_generator.py -t {{ (fernet_token_expiry | int) // 60 }} -i {{ groups['keystone'].index(inventory_hostname) }} -n {{ (groups['keystone'] | length) }}"
+  register: cron_jobs_json
+  when: keystone_token_provider == 'fernet'
+
+- name: Save the returned from cron jobs for building the crontab
+  set_fact:
+    cron_jobs: "{{ (cron_jobs_json.stdout | from_json).cron_jobs }}"
+  when: keystone_token_provider == 'fernet'
+
+- name: Copying files for keystone-fernet
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ node_config_directory }}/keystone-fernet/{{ item.dest }}"
+  with_items:
+    - { src: "crontab.j2", dest: "crontab" }
+    - { src: "fernet-rotate.sh.j2", dest: "fernet-rotate.sh" }
+    - { src: "fernet-node-sync.sh.j2", dest: "fernet-node-sync.sh" }
+  when: keystone_token_provider == 'fernet'
+
+- name: Copying files for keystone-ssh
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ node_config_directory }}/keystone-ssh/{{ item.dest }}"
+  with_items:
+    - { src: "sshd_config.j2", dest: "sshd_config" }
+    - { src: "id_rsa", dest: "id_rsa" }
+    - { src: "id_rsa.pub", dest: "id_rsa.pub" }
+    - { src: "ssh_config.j2", dest: "ssh_config" }
+  when: keystone_token_provider == 'fernet'

ansible/roles/keystone/tasks/deploy.yml

@@ -8,6 +8,11 @@
 - include: start.yml
   when: inventory_hostname in groups['keystone']
 
+- include: init_fernet.yml
+  when:
+    - inventory_hostname in groups['keystone']
+    - keystone_token_provider == 'fernet'
+
 - include: register.yml
   when: inventory_hostname in groups['keystone']
 

ansible/roles/keystone/tasks/do_reconfigure.yml

@@ -1,4 +1,17 @@
 ---
+- name: Set variable for keystone components used in reconfigure
+  set_fact:
+    keystone_items:
+      - { name: keystone, group: keystone }
+
+- name: Add fernet related components to variable if fernet is enabled
+  set_fact:
+    keystone_fernet_items:
+      - { name: keystone_fernet, group: keystone }
+      - { name: keystone_ssh, group: keystone }
+    keystone_items: "{{ keystone_items + keystone_fernet_items }}"
+  when: keystone_token_provider == 'fernet'
+
 - name: Ensuring the containers up
   kolla_docker:
     name: "{{ item.name }}"
@@ -6,8 +19,7 @@
   register: container_state
   failed_when: container_state.Running == false
   when: inventory_hostname in groups[item.group]
-  with_items:
+  with_items: keystone_items
-    - { name: keystone, group: keystone }
 
 - include: config.yml
 
@@ -17,8 +29,7 @@
   failed_when: false
   register: check_results
   when: inventory_hostname in groups[item.group]
-  with_items:
+  with_items: keystone_items
-    - { name: keystone, group: keystone }
 
 # NOTE(jeffrey4l): when config_strategy == 'COPY_ALWAYS'
 # and container env['KOLLA_CONFIG_STRATEGY'] == 'COPY_ONCE',
@@ -29,8 +40,7 @@
     action: "get_container_env"
   register: container_envs
   when: inventory_hostname in groups[item.group]
-  with_items:
+  with_items: keystone_items
-    - { name: keystone, group: keystone }
 
 - name: Remove the containers
   kolla_docker:
@@ -42,7 +52,7 @@
     - item[2]['rc'] == 1
     - inventory_hostname in groups[item[0]['group']]
   with_together:
-    - [{ name: keystone, group: keystone }]
+    - [keystone_items]
     - "{{ container_envs.results }}"
     - "{{ check_results.results }}"
 
@@ -59,6 +69,6 @@
     - item[2]['rc'] == 1
     - inventory_hostname in groups[item[0]['group']]
   with_together:
-    - [{ name: keystone, group: keystone }]
+    - [keystone_items]
     - "{{ container_envs.results }}"
     - "{{ check_results.results }}"

ansible/roles/keystone/tasks/init_fernet.yml

@@ -0,0 +1,15 @@
+---
+- name: Initialise fernet key authentication
+  command: "docker exec -t keystone_fernet kolla_keystone_bootstrap {{ keystone_username }} {{ keystone_groupname }}"
+  register: fernet_create
+  changed_when: "{{ fernet_create.stdout.find('localhost | SUCCESS => ') != -1 and (fernet_create.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  until: "(fernet_create.stdout.split()[2] == 'SUCCESS') or (fernet_create.stdout.find('Key repository is already initialized') != -1)"
+  retries: 10
+  delay: 5
+  run_once: True
+  delegate_to: "{{ groups['keystone'][0] }}"
+
+- name: Run key distribution
+  command: docker exec -t keystone_fernet /usr/bin/fernet-rotate.sh
+  run_once: True
+  delegate_to: "{{ groups['keystone'][0] }}"

ansible/roles/keystone/tasks/pull.yml

@@ -5,3 +5,21 @@
     common_options: "{{ docker_common_options }}"
     image: "{{ keystone_image_full }}"
   when: inventory_hostname in groups['keystone']
+
+- name: Pulling keystone_fernet image
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ keystone_fernet_image_full }}"
+  when:
+    - inventory_hostname in groups['keystone']
+    - keystone_token_provider == 'fernet'
+
+- name: Pulling keystone_ssh image
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ keystone_ssh_image_full }}"
+  when:
+    - inventory_hostname in groups['keystone']
+    - keystone_token_provider == 'fernet'

ansible/roles/keystone/tasks/start.yml

@@ -1,14 +1,49 @@
 ---
+- name: Set variable for inital keystone volumes
+  set_fact:
+    keystone_volumes:
+      - "{{ node_config_directory }}/keystone/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla/"
+
+- name: Add fernet volume to keystone volumes variable if fernet enabled
+  set_fact:
+    keystone_volumes: "{{ keystone_volumes + [\"keystone_fernet_tokens:/etc/keystone/fernet-keys\"] }}"
+  when: keystone_token_provider == 'fernet'
+
 - name: Starting keystone container
   kolla_docker:
     action: "start_container"
     common_options: "{{ docker_common_options }}"
     image: "{{ keystone_image_full }}"
     name: "keystone"
-    volumes:
+    volumes: "{{ keystone_volumes }}"
-      - "{{ node_config_directory }}/keystone/:{{ container_config_directory }}/:ro"
-      - "/etc/localtime:/etc/localtime:ro"
-      - "kolla_logs:/var/log/kolla/"
 
 - name: Wait for keystone startup
   wait_for: host={{ kolla_internal_fqdn }} port={{ keystone_admin_port }}
+
+- name: Starting keystone-ssh container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ keystone_ssh_image_full }}"
+    name: "keystone_ssh"
+    volumes:
+      - "{{ node_config_directory }}/keystone-ssh/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla/"
+      - "keystone_fernet_tokens:/etc/keystone/fernet-keys"
+  when: keystone_token_provider == 'fernet'
+
+- name: Starting keystone-fernet container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ keystone_fernet_image_full }}"
+    name: "keystone_fernet"
+    volumes:
+      - "{{ node_config_directory }}/keystone-fernet/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla/"
+      - "keystone_fernet_tokens:/etc/keystone/fernet-keys"
+  when: keystone_token_provider == 'fernet'

ansible/roles/keystone/templates/crontab.j2

@@ -0,0 +1,3 @@
+{% for cron_job in cron_jobs %}
+{{ cron_job['min'] }} {{ cron_job['hour'] }} * * {{ cron_job['day'] }} /usr/bin/fernet-rotate.sh
+{% endfor %}

ansible/roles/keystone/templates/fernet-node-sync.sh.j2

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Get data on the fernet tokens
+TOKEN_CHECK=$(/usr/bin/fetch_fernet_tokens.py -t {{ fernet_token_expiry }} -n {{ (groups['keystone'] | length) + 1 }})
+
+# Ensure the primary token exists and is not stale
+if $(echo "$TOKEN_CHECK" | grep -q '"update_required":"false"'); then
+    exit 0;
+fi
+
+# For each host node sync tokens
+{% for host in groups['keystone'] %}
+{% if inventory_hostname != host %}
+/usr/bin/rsync -azu --delete -e 'ssh -i /var/lib/keystone/.ssh/id_rsa -p {{ keystone_ssh_port }}' keystone@{{ host }}:/etc/keystone/fernet-keys/ /etc/keystone/fernet-keys
+{% endif %}
+{% endfor %}

ansible/roles/keystone/templates/fernet-rotate.sh.j2

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+keystone-manage --config-file /etc/keystone/keystone.conf fernet_rotate --keystone-user {{ keystone_username }} --keystone-group {{ keystone_groupname }}
+
+{% for host in groups['keystone'] %}
+{% if inventory_hostname != host %}
+/usr/bin/rsync -az -e 'ssh -i /var/lib/keystone/.ssh/id_rsa -p {{ keystone_ssh_port }}' --delete /etc/keystone/fernet-keys/ keystone@{{ host }}:/etc/keystone/fernet-keys
+{% endif %}
+{% endfor %}

ansible/roles/keystone/templates/id_rsa

@@ -0,0 +1 @@
+{{ keystone_ssh_key.private_key }}

ansible/roles/keystone/templates/id_rsa.pub

@@ -0,0 +1 @@
+{{ keystone_ssh_key.public_key }}

ansible/roles/keystone/templates/keystone-fernet.json.j2

@@ -0,0 +1,23 @@
+{% set cron_cmd = 'cron -f' if kolla_base_distro in ['ubuntu', 'debian'] else 'crond -s -n' %}
+{
+    "command": "{{ cron_cmd }}",
+    "config_files": [{
+            "source": "{{ container_config_directory }}/crontab",
+            "dest": "/var/spool/cron/crontabs/root/fernet-cron",
+            "owner": "root",
+            "perm": "0644"
+        },
+        {
+            "source": "{{ container_config_directory }}/fernet-rotate.sh",
+            "dest": "/usr/bin/fernet-rotate.sh",
+            "owner": "root",
+            "perm": "0755"
+        },
+        {
+            "source": "{{ container_config_directory }}/fernet-node-sync.sh",
+            "dest": "/usr/bin/fernet-node-sync.sh",
+            "owner": "root",
+            "perm": "0755"
+        }
+    ]
+}

ansible/roles/keystone/templates/keystone-ssh.json.j2

@@ -0,0 +1,29 @@
+{
+    "command": "/usr/sbin/sshd -D",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/sshd_config",
+            "dest": "/etc/ssh/sshd_config",
+            "owner": "root",
+            "perm": "0644"
+        },
+        {
+            "source": "{{ container_config_directory }}/ssh_config",
+            "dest": "/var/lib/keystone/.ssh/config",
+            "owner": "keystone",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/id_rsa",
+            "dest": "/var/lib/keystone/.ssh/id_rsa",
+            "owner": "keystone",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/id_rsa.pub",
+            "dest": "/var/lib/keystone/.ssh/authorized_keys",
+            "owner": "keystone",
+            "perm": "0600"
+        }
+    ]
+}

ansible/roles/keystone/templates/keystone.conf.j2

@@ -16,6 +16,15 @@ domain_specific_drivers_enabled = true
 domain_config_dir = /etc/keystone/domains
 {% endif %}
 
+{% if keystone_token_provider == 'fernet' %}
+[token]
+provider = {{ keystone_token_provider }}
+expiration = {{ fernet_token_expiry }}
+
+[fernet_tokens]
+max_active_keys = {{ (groups['keystone'] | length) + 1 }}
+{% endif %}
+
 [cache]
 backend = oslo_cache.memcache_pool
 enabled = True

ansible/roles/keystone/templates/ssh_config.j2

@@ -0,0 +1,4 @@
+Host {% for host in groups['keystone'] %}{% if inventory_hostname != host %}{{ host }} {% endif %}{% endfor %}
+  StrictHostKeyChecking no
+  UserKnownHostsFile /dev/null
+  Port {{ keystone_ssh_port }}

ansible/roles/keystone/templates/sshd_config.j2

@@ -0,0 +1,5 @@
+Port {{ keystone_ssh_port }}
+ListenAddress {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
+
+SyslogFacility AUTHPRIV
+UsePAM yes

ansible/roles/prechecks/tasks/service_checks.yml

@@ -42,3 +42,9 @@
   register: result
   changed_when: false
   failed_when: result.stdout | regex_replace('(.*ssh_key.*)', '') | search(":")
+
+- name: Checking fernet_token_expiry in globals.yml. Update fernet_token_expiry to allowed value if this task fails
+  local_action: command grep '^[^#]*fernet_token_expiry:\s*\d*' "{{ CONFIG_DIR }}/globals.yml" | sed 's/[^0-9]*//g'
+  register: result
+  changed_when: false
+  failed_when: result.stdout | regex_replace('(60|120|180|240|300|360|600|720|900|1200|1800|3600|7200|10800|14400|21600|28800|43200|86400|604800)', '') | search(".+")

etc/kolla/globals.yml

@@ -105,6 +105,15 @@ neutron_external_interface: "eth1"
 # Valid options are [ novnc, spice ]
 #nova_console: "novnc"
 
+# Valid options are [ uuid, fernet ]
+#keystone_token_provider: 'uuid'
+# Interval to rotate fernet keys by (in seconds). Must be an interval of
+# 60(1 min), 120(2 min), 180(3 min), 240(4 min), 300(5 min), 360(6 min),
+# 600(10 min), 720(12 min), 900(15 min), 1200(20 min), 1800(30 min),
+# 3600(1 hour), 7200(2 hour), 10800(3 hour), 14400(4 hour), 21600(6 hour),
+# 28800(8 hour), 43200(12 hour), 86400(1 day), 604800(1 week).
+#fernet_token_expiry: 86400
+
 # OpenStack services can be enabled or disabled with these options
 #enable_ceilometer: "no"
 #enable_central_logging: "no"

etc/kolla/passwords.yml

@@ -81,6 +81,10 @@ kolla_ssh_key:
   private_key:
   public_key:
 
+keystone_ssh_key:
+  private_key:
+  public_key:
+
 ####################
 # RabbitMQ options
 ####################

kolla/cmd/genpwd.py

@@ -43,7 +43,7 @@ def main():
     uuid_keys = ['ceph_cluster_fsid', 'rbd_secret_uuid']
 
     # SSH key pair
-    ssh_keys = ['kolla_ssh_key', 'nova_ssh_key']
+    ssh_keys = ['kolla_ssh_key', 'nova_ssh_key', 'keystone_ssh_key']
 
     # If these keys are None, leave them as None
     blank_keys = ['docker_registry_password']

releasenotes/notes/add-fernet-support-54ccb88b901d8d8b.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  - Add full support for fernet with distributed token node syncing