From f87b238db52353884dffd60b86dea78c5139ebd4 Mon Sep 17 00:00:00 2001
From: Doug Szumski <doug@stackhpc.com>
Date: Fri, 23 Feb 2018 17:36:27 +0000
Subject: [PATCH] Add support for deploying ZooKeeper

This picks up the abandoned review:
https://review.openstack.org/#/c/412423

Co-Authored-By: Sam Yaple <sam@yaple.net>
Partially-Implements: blueprint monasca-roles
Change-Id: Idc01afcb125271181ee5e15d327d0929f07b49e9
---
 ansible/group_vars/all.yml                    |  5 ++
 ansible/inventory/all-in-one                  |  3 +
 ansible/inventory/multinode                   |  3 +
 ansible/roles/common/tasks/config.yml         |  2 +
 .../conf/filter/00-record_transformer.conf.j2 |  7 ++
 .../templates/conf/input/06-zookeeper.conf.j2 | 11 +++
 .../cron-logrotate-zookeeper.conf.j2          |  3 +
 ansible/roles/common/templates/cron.json.j2   |  1 +
 .../roles/common/templates/fluentd.json.j2    |  6 ++
 ansible/roles/zookeeper/defaults/main.yml     | 28 +++++++
 ansible/roles/zookeeper/handlers/main.yml     | 24 ++++++
 ansible/roles/zookeeper/meta/main.yml         |  3 +
 ansible/roles/zookeeper/tasks/check.yml       |  1 +
 ansible/roles/zookeeper/tasks/config.yml      | 76 +++++++++++++++++++
 ansible/roles/zookeeper/tasks/deploy.yml      |  5 ++
 ansible/roles/zookeeper/tasks/main.yml        |  2 +
 ansible/roles/zookeeper/tasks/precheck.yml    | 21 +++++
 ansible/roles/zookeeper/tasks/pull.yml        | 10 +++
 ansible/roles/zookeeper/tasks/reconfigure.yml |  2 +
 ansible/roles/zookeeper/tasks/upgrade.yml     |  5 ++
 ansible/roles/zookeeper/templates/myid.j2     |  5 ++
 .../zookeeper/templates/zookeeper.cfg.j2      |  8 ++
 .../zookeeper/templates/zookeeper.json.j2     | 29 +++++++
 ansible/site.yml                              |  9 +++
 etc/kolla/globals.yml                         |  1 +
 .../add-zookeeper-role-9eb474f26035ec77.yaml  |  5 ++
 26 files changed, 275 insertions(+)
 create mode 100644 ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2
 create mode 100644 ansible/roles/common/templates/cron-logrotate-zookeeper.conf.j2
 create mode 100644 ansible/roles/zookeeper/defaults/main.yml
 create mode 100644 ansible/roles/zookeeper/handlers/main.yml
 create mode 100644 ansible/roles/zookeeper/meta/main.yml
 create mode 100644 ansible/roles/zookeeper/tasks/check.yml
 create mode 100644 ansible/roles/zookeeper/tasks/config.yml
 create mode 100644 ansible/roles/zookeeper/tasks/deploy.yml
 create mode 100644 ansible/roles/zookeeper/tasks/main.yml
 create mode 100644 ansible/roles/zookeeper/tasks/precheck.yml
 create mode 100644 ansible/roles/zookeeper/tasks/pull.yml
 create mode 100644 ansible/roles/zookeeper/tasks/reconfigure.yml
 create mode 100644 ansible/roles/zookeeper/tasks/upgrade.yml
 create mode 100644 ansible/roles/zookeeper/templates/myid.j2
 create mode 100644 ansible/roles/zookeeper/templates/zookeeper.cfg.j2
 create mode 100644 ansible/roles/zookeeper/templates/zookeeper.json.j2
 create mode 100644 releasenotes/notes/add-zookeeper-role-9eb474f26035ec77.yaml

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index b5ccbb3d5e..81b8165391 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -303,6 +303,10 @@ trove_api_port: "8779"
 
 watcher_api_port: "9322"
 
+zookeeper_client_port: "2181"
+zookeeper_peer_port: "2888"
+zookeeper_quorum_port: "3888"
+
 zun_api_port: "9517"
 
 opendaylight_clustering_port: "2550"
@@ -477,6 +481,7 @@ enable_trove: "no"
 enable_vitrage: "no"
 enable_vmtp: "no"
 enable_watcher: "no"
+enable_zookeeper: "no"
 enable_zun: "no"
 
 ovs_datapath: "{{ 'netdev' if enable_ovs_dpdk | bool else 'system' }}"
diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one
index 31d1c0db43..fd50b41025 100644
--- a/ansible/inventory/all-in-one
+++ b/ansible/inventory/all-in-one
@@ -222,6 +222,9 @@ control
 [bifrost:children]
 deployment
 
+[zookeeper:children]
+control
+
 [zun:children]
 control
 
diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode
index 29932f31f7..d2ba5b8b4c 100644
--- a/ansible/inventory/multinode
+++ b/ansible/inventory/multinode
@@ -241,6 +241,9 @@ control
 [bifrost:children]
 deployment
 
+[zookeeper:children]
+control
+
 [zun:children]
 control
 
diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml
index 2dba74b3ba..89531bb730 100644
--- a/ansible/roles/common/tasks/config.yml
+++ b/ansible/roles/common/tasks/config.yml
@@ -51,6 +51,7 @@
     - "03-rabbitmq"
     - "04-openstack-wsgi"
     - "05-libvirt"
+    - "06-zookeeper"
   notify:
     - Restart fluentd container
 
@@ -217,6 +218,7 @@
     - { name: "trove", enabled: "{{ enable_trove }}" }
     - { name: "vitrage", enabled: "{{ enable_vitrage }}" }
     - { name: "watcher", enabled: "{{ enable_watcher }}" }
+    - { name: "zookeeper", enabled: "{{ enable_zookeeper }}" }
     - { name: "zun", enabled: "{{ enable_zun }}" }
   notify:
     - Restart cron container
diff --git a/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2
index 12b9b566c9..37ea4c19e6 100644
--- a/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2
+++ b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2
@@ -7,6 +7,13 @@
     </record>
 </filter>
 
+<filter infra.var.log.kolla.*.*.log>
+    @type record_transformer
+    <record>
+        Logger ${tag_parts[4]}
+    </record>
+</filter>
+
 <filter infra.*>
     @type record_transformer
     <record>
diff --git a/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 b/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2
new file mode 100644
index 0000000000..e80b8a3c32
--- /dev/null
+++ b/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2
@@ -0,0 +1,11 @@
+{% set fluentd_dir = 'td-agent' if kolla_base_distro in ['ubuntu', 'debian'] else 'fluentd' %}
+<source>
+  @type tail
+  path /var/log/kolla/zookeeper/zookeeper.log
+  pos_file /var/run/{{ fluentd_dir }}/zookeeper.pos
+  tag infra.*
+  format multiline
+  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \S+ \S+ .*$/
+  format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}) \[(?<server_id>\S+)\] \S+ (?<log_level>\S+) (?<Payload>.*)$/
+  time_key Timestamp
+</source>
diff --git a/ansible/roles/common/templates/cron-logrotate-zookeeper.conf.j2 b/ansible/roles/common/templates/cron-logrotate-zookeeper.conf.j2
new file mode 100644
index 0000000000..b2d07766e2
--- /dev/null
+++ b/ansible/roles/common/templates/cron-logrotate-zookeeper.conf.j2
@@ -0,0 +1,3 @@
+"/var/log/kolla/zookeeper/*.log"
+{
+}
diff --git a/ansible/roles/common/templates/cron.json.j2 b/ansible/roles/common/templates/cron.json.j2
index 8601208172..bfe3c4c1c4 100644
--- a/ansible/roles/common/templates/cron.json.j2
+++ b/ansible/roles/common/templates/cron.json.j2
@@ -53,6 +53,7 @@
     ( 'trove', enable_trove ),
     ( 'vitrage', enable_vitrage ),
     ( 'watcher', enable_watcher ),
+    ( 'zookeeper', enable_zookeeper ),
     ( 'zun', enable_zun )
 ] %}
 {
diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2
index f711699444..c622b0a664 100644
--- a/ansible/roles/common/templates/fluentd.json.j2
+++ b/ansible/roles/common/templates/fluentd.json.j2
@@ -47,6 +47,12 @@
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },
+        {
+            "source": "{{ container_config_directory }}/input/06-zookeeper.conf",
+            "dest": "{{ fluentd_dir }}/input/06-zookeeper.conf",
+            "owner": "{{ fluentd_user }}",
+            "perm": "0600"
+        },
         {# Copy all configuration files in filter/ directory to include #}
         {# custom filter configs. #}
         {
diff --git a/ansible/roles/zookeeper/defaults/main.yml b/ansible/roles/zookeeper/defaults/main.yml
new file mode 100644
index 0000000000..125165e729
--- /dev/null
+++ b/ansible/roles/zookeeper/defaults/main.yml
@@ -0,0 +1,28 @@
+---
+zookeeper_services:
+  zookeeper:
+    container_name: zookeeper
+    group: zookeeper
+    enabled: true
+    image: "{{ zookeeper_image_full }}"
+    environment:
+      ZOO_LOG_DIR: /var/log/kolla/zookeeper
+      ZOO_LOG4J_PROP: "{{ zookeeper_log_settings }}"
+    volumes:
+      - "{{ node_config_directory }}/zookeeper/:{{ container_config_directory }}/"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "zookeeper:/var/lib/zookeeper/data"
+      - "kolla_logs:/var/log/kolla/"
+
+####################
+# Zookeeper
+####################
+zookeeper_log_settings: 'INFO,ROLLINGFILE'
+
+####################
+# Docker
+####################
+zookeeper_install_type: "{{ kolla_install_type }}"
+zookeeper_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ zookeeper_install_type }}-zookeeper"
+zookeeper_tag: "{{ openstack_release }}"
+zookeeper_image_full: "{{ zookeeper_image }}:{{ zookeeper_tag }}"
diff --git a/ansible/roles/zookeeper/handlers/main.yml b/ansible/roles/zookeeper/handlers/main.yml
new file mode 100644
index 0000000000..3ba27a0c15
--- /dev/null
+++ b/ansible/roles/zookeeper/handlers/main.yml
@@ -0,0 +1,24 @@
+---
+- name: Restart zookeeper container
+  vars:
+    service_name: "zookeeper"
+    service: "{{ zookeeper_services[service_name] }}"
+    config_json: "{{ zookeeper_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    zookeeper_conf: "{{ zookeeper_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    zookeeper_instance_id: "{{ zookeeper_instance_id.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    zookeeper_container: "{{ check_zookeeper_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    environment: "{{ service.environment }}"
+    volumes: "{{ service.volumes }}"
+  when:
+    - action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or zookeeper_conf.changed | bool
+      or zookeeper_instance_id.changed | bool
+      or zookeeper_container.changed | bool
diff --git a/ansible/roles/zookeeper/meta/main.yml b/ansible/roles/zookeeper/meta/main.yml
new file mode 100644
index 0000000000..6b4fff8fef
--- /dev/null
+++ b/ansible/roles/zookeeper/meta/main.yml
@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: common }
diff --git a/ansible/roles/zookeeper/tasks/check.yml b/ansible/roles/zookeeper/tasks/check.yml
new file mode 100644
index 0000000000..ed97d539c0
--- /dev/null
+++ b/ansible/roles/zookeeper/tasks/check.yml
@@ -0,0 +1 @@
+---
diff --git a/ansible/roles/zookeeper/tasks/config.yml b/ansible/roles/zookeeper/tasks/config.yml
new file mode 100644
index 0000000000..b69c5b19e5
--- /dev/null
+++ b/ansible/roles/zookeeper/tasks/config.yml
@@ -0,0 +1,76 @@
+---
+- name: Ensuring config directories exist
+  file:
+    path: "{{ node_config_directory }}/{{ item.key }}"
+    state: "directory"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+    mode: "0770"
+    recurse: yes
+  become: true
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ zookeeper_services }}"
+
+- name: Copying over config.json files for services
+  template:
+    src: "{{ item.key }}.json.j2"
+    dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
+    mode: "0660"
+  become: true
+  register: zookeeper_config_jsons
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ zookeeper_services }}"
+  notify:
+    - Restart zookeeper container
+
+- name: Copying over zookeeper configuration
+  merge_configs:
+    sources:
+      - "{{ role_path }}/templates/{{ item.key }}.cfg.j2"
+      - "{{ node_custom_config }}/{{ item.key }}.cfg"
+      - "{{ node_custom_config }}/{{ item.key }}/{{ inventory_hostname }}/{{ item.key }}.cfg"
+    dest: "{{ node_config_directory }}/{{ item.key }}/{{ item.key }}.cfg"
+    mode: "0660"
+  become: true
+  register: zookeeper_confs
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ zookeeper_services }}"
+  notify:
+    - Restart zookeeper container
+
+- name: Copying over zookeeper instance id
+  template:
+    src: "myid.j2"
+    dest: "{{ node_config_directory }}/{{ item.key }}/myid"
+    mode: "0660"
+  become: true
+  register: zookeeper_instance_id
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ zookeeper_services }}"
+  notify:
+    - Restart zookeeper container
+
+- name: Check zookeeper containers
+  kolla_docker:
+    action: "compare_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ item.value.container_name }}"
+    image: "{{ item.value.image }}"
+    volumes: "{{ item.value.volumes }}"
+    environment: "{{ item.value.environment }}"
+  register: check_zookeeper_containers
+  when:
+    - action != "config"
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ zookeeper_services }}"
+  notify:
+    - Restart zookeeper container
diff --git a/ansible/roles/zookeeper/tasks/deploy.yml b/ansible/roles/zookeeper/tasks/deploy.yml
new file mode 100644
index 0000000000..dd26ecc34d
--- /dev/null
+++ b/ansible/roles/zookeeper/tasks/deploy.yml
@@ -0,0 +1,5 @@
+---
+- include: config.yml
+
+- name: Flush handlers
+  meta: flush_handlers
diff --git a/ansible/roles/zookeeper/tasks/main.yml b/ansible/roles/zookeeper/tasks/main.yml
new file mode 100644
index 0000000000..b017e8b4ad
--- /dev/null
+++ b/ansible/roles/zookeeper/tasks/main.yml
@@ -0,0 +1,2 @@
+---
+- include: "{{ action }}.yml"
diff --git a/ansible/roles/zookeeper/tasks/precheck.yml b/ansible/roles/zookeeper/tasks/precheck.yml
new file mode 100644
index 0000000000..38aeeb95c8
--- /dev/null
+++ b/ansible/roles/zookeeper/tasks/precheck.yml
@@ -0,0 +1,21 @@
+---
+- name: Get container facts
+  kolla_container_facts:
+    name:
+      - zookeeper
+  register: container_facts
+
+- name: Checking zookeeper ports are available
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ item }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  with_items:
+    - "{{ zookeeper_client_port }}"
+    - "{{ zookeeper_peer_port }}"
+    - "{{ zookeeper_quorum_port }}"
+  when:
+    - container_facts['zookeeper'] is not defined
+    - inventory_hostname in groups['zookeeper']
diff --git a/ansible/roles/zookeeper/tasks/pull.yml b/ansible/roles/zookeeper/tasks/pull.yml
new file mode 100644
index 0000000000..f5614aca34
--- /dev/null
+++ b/ansible/roles/zookeeper/tasks/pull.yml
@@ -0,0 +1,10 @@
+---
+- name: Pulling zookeeper images
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ item.value.image }}"
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ zookeeper_services }}"
diff --git a/ansible/roles/zookeeper/tasks/reconfigure.yml b/ansible/roles/zookeeper/tasks/reconfigure.yml
new file mode 100644
index 0000000000..e078ef1318
--- /dev/null
+++ b/ansible/roles/zookeeper/tasks/reconfigure.yml
@@ -0,0 +1,2 @@
+---
+- include: deploy.yml
diff --git a/ansible/roles/zookeeper/tasks/upgrade.yml b/ansible/roles/zookeeper/tasks/upgrade.yml
new file mode 100644
index 0000000000..dd26ecc34d
--- /dev/null
+++ b/ansible/roles/zookeeper/tasks/upgrade.yml
@@ -0,0 +1,5 @@
+---
+- include: config.yml
+
+- name: Flush handlers
+  meta: flush_handlers
diff --git a/ansible/roles/zookeeper/templates/myid.j2 b/ansible/roles/zookeeper/templates/myid.j2
new file mode 100644
index 0000000000..4492f393cb
--- /dev/null
+++ b/ansible/roles/zookeeper/templates/myid.j2
@@ -0,0 +1,5 @@
+{% for host in groups['zookeeper'] -%}
+{% if hostvars[host]['ansible_hostname'] == ansible_hostname -%}
+{{ loop.index }}
+{%- endif %}
+{%- endfor %}
diff --git a/ansible/roles/zookeeper/templates/zookeeper.cfg.j2 b/ansible/roles/zookeeper/templates/zookeeper.cfg.j2
new file mode 100644
index 0000000000..f7ab93d647
--- /dev/null
+++ b/ansible/roles/zookeeper/templates/zookeeper.cfg.j2
@@ -0,0 +1,8 @@
+tickTime=2000
+initLimit=10
+syncLimit=5
+dataDir=/var/lib/zookeeper/data
+clientPort={{ zookeeper_client_port }}
+{% for host in groups['zookeeper'] %}
+server.{{ loop.index }}={{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_peer_port }}:{{ zookeeper_quorum_port }}
+{% endfor %}
diff --git a/ansible/roles/zookeeper/templates/zookeeper.json.j2 b/ansible/roles/zookeeper/templates/zookeeper.json.j2
new file mode 100644
index 0000000000..9d9d609901
--- /dev/null
+++ b/ansible/roles/zookeeper/templates/zookeeper.json.j2
@@ -0,0 +1,29 @@
+{
+    "command": "/opt/zookeeper/bin/zkServer.sh start-foreground /etc/zookeeper/conf/zoo.cfg",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/myid",
+            "dest": "/var/lib/zookeeper/data/myid",
+            "owner": "zookeeper",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/zookeeper.cfg",
+            "dest": "/etc/zookeeper/conf/zoo.cfg",
+            "owner": "zookeeper",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/lib/zookeeper",
+            "owner": "zookeeper:zookeeper",
+            "recurse": true
+        },
+        {
+            "path": "/var/log/kolla/zookeeper",
+            "owner": "zookeeper:zookeeper",
+            "recurse": true
+        }
+    ]
+}
diff --git a/ansible/site.yml b/ansible/site.yml
index 4c17417920..a599338171 100644
--- a/ansible/site.yml
+++ b/ansible/site.yml
@@ -75,6 +75,15 @@
         tags: collectd,
         when: enable_collectd | bool }
 
+- name: Apply role zookeeper
+  gather_facts: false
+  hosts: zookeeper
+  serial: '{{ serial|default("0") }}'
+  roles:
+    - { role: zookeeper,
+        tags: zookeeper,
+        when: enable_zookeeper | bool }
+
 - name: Apply role elasticsearch
   gather_facts: false
   hosts: elasticsearch
diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml
index 1b57792752..067739de7a 100644
--- a/etc/kolla/globals.yml
+++ b/etc/kolla/globals.yml
@@ -234,6 +234,7 @@ kolla_internal_vip_address: "10.10.10.254"
 #enable_vitrage: "no"
 #enable_vmtp: "no"
 #enable_watcher: "no"
+#enable_zookeeper: "no"
 #enable_zun: "no"
 
 ##############
diff --git a/releasenotes/notes/add-zookeeper-role-9eb474f26035ec77.yaml b/releasenotes/notes/add-zookeeper-role-9eb474f26035ec77.yaml
new file mode 100644
index 0000000000..84f0cd38ba
--- /dev/null
+++ b/releasenotes/notes/add-zookeeper-role-9eb474f26035ec77.yaml
@@ -0,0 +1,5 @@
+---
+features:
+  - Add a role for deploying Apache Zookeeper for the purpose of
+    supporting Apache Kafka. See https://zookeeper.apache.org/
+    for more details.