adding kolide fleet

* adds kolide fleet
  * integrates osquery to kolide fleet server

Change-Id: I646364c44bb99d4397bb35068600c49b7bfd62c2

osquery/ansible-role-requirements.yml

@@ -0,0 +1,29 @@
+---
+#- name: systemd_service
+#  scm: git
+#  src: https://git.openstack.org/openstack/ansible-role-systemd_service
+#  version: master
+- name: config_template
+  scm: git
+  src: https://git.openstack.org/openstack/ansible-config_template
+  version: master
+- name: osquery
+  scm: git
+  src: https://github.com/juju4/ansible-osquery
+  version: master
+- name: redis
+  scm: git
+  src: https://github.com/geerlingguy/ansible-role-redis
+  version: master
+- name: mariadb
+  scm: git
+  src: https://github.com/lechuckroh/ansible-role-mariadb
+  version: master
+- name: galera_client
+  scm: git
+  src: https://git.openstack.org/openstack/openstack-ansible-galera_client
+  version: master
+- name: galera_server
+  scm: git
+  src: https://git.openstack.org/openstack/openstack-ansible-galera_server
+  version: master

osquery/assets/place-holder.svg

@@ -0,0 +1 @@
+<svg width="800" height="800" xmlns="http://www.w3.org/2000/svg"><rect x="2" y="2" width="796" height="796" style="fill:#dedede;stroke:#555555;stroke-width:2"/><text x="50%" y="50%" font-size="18" text-anchor="middle" alignment-baseline="middle" font-family="monospace, sans-serif" fill="#555555">placeholder</text></svg>

osquery/conf.d/fleet.yml

@@ -0,0 +1,6 @@
+fleet_hosts:
+  logging1:
+    ip: 172.22.8.27
+mariadb_hosts:
+  logging1:
+    ip: 172.22.8.27

osquery/env.d/fleet.yml

@@ -0,0 +1,37 @@
+---
+component_skel:
+  fleet:
+    belongs_to:
+      - fleet_all
+  mariadb:
+    belongs_to:
+      - fleet_all
+  osquery:
+    belogs_to:
+      - all
+
+container_skel:
+  mariadb_container:
+    belongs_to:
+      - mariadb_containers
+    contains:
+      - mariadb
+  fleet_container:
+    belongs_to:
+      - fleet_containers
+    contains:
+      - fleet
+
+physical_skel:
+  mariadb_container:
+    belongs_to:
+      - all_containers
+  mariadb_hosts:
+    belongs_to:
+      - hosts
+  fleet_containers:
+    belongs_to:
+      - all_containers
+  fleet_hosts:
+    belongs_to:
+      - hosts

osquery/fleetConfig.yml

@@ -0,0 +1,26 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: create fleet dir
+  file:
+    path: /etc/fleet
+    state: directory
+
+- name: Drop fleet conf file
+  template:
+    src: templates/fleet_config.yml.j2
+    dest: /etc/fleet/fleet_config.yml
+  tags:
+    - fleet_config

osquery/fleetGetEnrollmentToken.yml

@@ -0,0 +1,22 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: retrieve Enrollment Token
+  command: /usr/local/bin/fleetctl get enroll-secret
+  register: _enrollment_token
+
+- name: Set kolide fleet enrollment token fact
+  set_fact:
+    kolide_fleet_enroll_secret: "{{ _enrollment_token.stdout }}"

osquery/fleetMigrateDB.yml

@@ -0,0 +1,18 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+- name: Migrate the fleet database
+  command: /usr/local/bin/fleet prepare db --config=/etc/fleet/fleet_config.yml  --no-prompt

osquery/fleetRegisterAdmin.yml

@@ -0,0 +1,32 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: check to see if we have alredy registered fleetctl admin
+  stat:
+    path: ~/.fleet/config
+  register: fleet_config_set
+
+- name: set fleetctl default context
+  command: fleetctl config set --address https://localhost:{{ kolide_fleet_port }} --tls-skip-verify
+  when: fleet_config_set.stat.exists == false
+
+- name: register admin account
+  command: /usr/local/bin/fleetctl setup --email {{ kolide_fleet_admin_email }} --password {{ kolide_fleet_admin_password }}
+  register: fleet_register_admin
+  ignore_errors: true # ignore errors as we might have already set this it will be caought later
+
+- name: login admin account
+  command: /usr/local/bin/fleetctl login --email {{ kolide_fleet_admin_email }} --password {{ kolide_fleet_admin_password }}
+  when: fleet_register_admin['stderr'] == "Kolide Fleet has already been setup"

osquery/fleetRequirements.yml

@@ -0,0 +1,40 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#- name:    Install prerequisites
+#  apt:     name= {{item}} update_cache=yes
+#  with_items:
+#   - apt-transport-https
+#   - ca-certificates
+#   - curl
+#   - software-properties-common
+- name: Run the systemd service role
+  include_role:
+    name: redis
+    private: true
+- name: create fleet dir
+  file:
+    path: /etc/fleet/ssl
+    state: directory
+
+- name: Debug fleet_config
+  debug:
+    var:  hostvars[groups['mariadb'][0]]['ansible_host']
+    verbosity: 2
+
+- name: Drop fleet conf file
+  template:
+    src: templates/fleet_config.yml.j2
+    dest: /etc/fleet/fleet_config.yml

osquery/fleetSSL.yml

@@ -0,0 +1,27 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Ensure the private ssl directory exists
+  file:
+    dest: "/etc/ssl/private"
+    state: "directory"
+  tags:
+    - fleet-ssl
+
+- include_tasks: fleetSSLselfSigned.yml
+  when:
+    - kolide_fleet_user_ssl_cert is not defined or kolide_fleet_user_ssl_key is not defined
+
+- include_tasks: fleetSSLuserProvided.yml

osquery/fleetSSLdistribute.yml

@@ -0,0 +1,30 @@
+---
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Distribute self signed ssl key
+  copy:
+    dest: "{{ kolide_fleet_ssl_key }}"
+    content: "{{ hostvars[groups['fleet'][0]]['kolide_fleet_ssl_key_fact'] | b64decode }}"
+    mode: "0640"
+  tags:
+    - fleet-ssl
+
+- name: Distribute self signed ssl cert
+  copy:
+    dest: "{{ kolide_fleet_ssl_cert }}"
+    content: "{{ hostvars[groups['fleet'][0]]['kolide_fleet_ssl_cert_fact'] | b64decode }}"
+    mode: "0640"
+  tags:
+    - fleet-ssl

osquery/fleetSSLkeyCreate.yml

@@ -0,0 +1,37 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Remove self signed certs and keys for regen
+  file:
+    dest: "{{ item }}"
+    state: "absent"
+  with_items:
+    - "{{ kolide_fleet_ssl_pem }}"
+    - "{{ kolide_fleet_ssl_key }}"
+    - "{{ kolide_fleet_ssl_cert }}"
+  tags:
+    - fleet-ssl
+
+- name: Create self-signed ssl cert
+  command: >
+    openssl req -new -nodes -sha256 -x509 -subj
+    "{{ kolide_fleet_ssl_self_signed_subject }}"
+    -days 3650
+    -keyout {{ kolide_fleet_ssl_key }}
+    -out {{ kolide_fleet_ssl_cert }}
+    -extensions v3_ca
+    creates={{ kolide_fleet_ssl_cert }}
+  tags:
+    - kolide_fleet-ssl

osquery/fleetSSLselfSigned.yml

@@ -0,0 +1,18 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- include_tasks: fleetSSLkeyCreate.yml
+- include_tasks: fleetSSLstore.yml
+- include_tasks: fleetSSLdistribute.yml

osquery/fleetSSLstore.yml

@@ -0,0 +1,29 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Store ssl cert
+  slurp:
+    src: "{{ kolide_fleet_ssl_cert }}"
+  register: _kolide_fleet_ssl_cert
+
+- name: Store ssl key
+  slurp:
+    src: "{{ kolide_fleet_ssl_key }}"
+  register: _kolide_fleet_ssl_key
+
+- name: Register a fact for the cert and key
+  set_fact:
+    kolide_fleet_ssl_cert_fact: "{{ _kolide_fleet_ssl_cert.content }}"
+    kolide_fleet_ssl_key_fact: "{{ _kolide_fleet_ssl_key.content }}"

osquery/fleetSSLuserProvided.yml

@@ -0,0 +1,47 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Drop user provided ssl cert
+  copy:
+    src: "{{ kolide_fleet_user_ssl_cert }}"
+    dest: "{{ kolide_fleet_ssl_cert }}"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  when: kolide_fleet_user_ssl_cert is defined
+  tags:
+    - fleet-ssl
+
+- name: Drop user provided ssl key
+  copy:
+    src: "{{ kolide_fleet_user_ssl_key }}"
+    dest: "{{ kolide_fleet_ssl_key }}"
+    owner: "root"
+    group: "root"
+    mode: "0640"
+  when: kolide_fleet_user_ssl_key is defined
+  tags:
+    - fleet-ssl
+
+- name: Drop user provided ssl CA cert
+  copy:
+    src: "{{ kolide_fleet_user_ssl_ca_cert }}"
+    dest: "{{ kolide_fleet_ssl_ca_cert }}"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  when: kolide_fleet_user_ssl_ca_cert is defined
+  tags:
+    - fleet-ssl

osquery/fleetServerInstall.yml

@@ -0,0 +1,50 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name:    Install prerequisites
+  apt:     name={{ item }} update_cache=yes
+  with_items:
+   - unzip
+   - ca-certificates
+   - apt-transport-https
+   - software-properties-common
+   - curl
+- name: Run the systemd service role
+  include_role:
+    name: redis
+    private: true
+
+- name: create fleet dir
+  file:
+    path: '/tmp/fleet_{{ kolide_fleet_version }}'
+    state: directory
+
+- name: Unarchive Fleet binaries
+  unarchive:
+    src: '{{ kolide_fleet_url  }}/{{ kolide_fleet_version }}/fleet_{{ kolide_fleet_version }}.zip'
+    dest: '/tmp/fleet_{{ kolide_fleet_version}}/'
+    remote_src: yes
+
+- name: Copy unarchived binaries
+  copy:
+    src: '/tmp/fleet_{{ kolide_fleet_version }}/linux/{{ item }}'
+    dest: '/usr/local/bin/'
+    mode: '0755'
+    owner: 'root'
+    group: 'root'
+    remote_src: yes
+  with_items:
+  - 'fleet'
+  - 'fleetctl'

osquery/fleetService.yml

@@ -0,0 +1,31 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Run the systemd service role
+  include_role:
+    name: systemd_service
+    private: true
+  vars:
+    systemd_service_restart_changed: false
+    systemd_services:
+      - service_name: "kolide-fleet"
+        execstarts:
+          - /usr/local/bin/fleet serve --config=/etc/fleet/fleet_config.yml
+        config_overrides:
+          Unit:
+            Wants: network-online.target
+            Requires: redis-server.service
+  tags:
+    - server-install

osquery/fleetStartService.yml

@@ -0,0 +1,45 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+
+- name: Start redis fleet kolide service - initd
+  service:
+    name: redis
+    enabled: yes
+    state: restarted
+  when:
+    - ansible_service_mgr == "init"
+  tags:
+    - config
+- name: Start fleet kolide service - initd
+  service:
+    name: redis
+    enabled: yes
+    state: restarted
+  when:
+    - ansible_service_mgr == "init"
+  tags:
+    - config
+
+- name: Enable and restart kolide - systemd
+  systemd:
+    name: "kolide-fleet"
+    enabled: true
+    state: restarted
+  when:
+    - ansible_service_mgr == "systemd"
+  tags:
+    - config

osquery/haproxy.example

@@ -0,0 +1,9 @@
+haproxy_extra_services:
+  - service:
+      haproxy_service_name: kolide-fleet
+      haproxy_ssl: False
+      haproxy_backend_nodes: "{{ groups['fleet'] | default([]) }}"  # Fleet nodes
+      haproxy_port: 8443  # This is set using the "kolide_fleet_port" variable
+      haproxy_check_port: 443  # This is set using the "kolide_fleet_port" variable
+      haproxy_backend_port: 443  # This is set using the "kolide_fleet_port" variable
+      haproxy_balance_type: tcp

osquery/installKolideFleet.yml

@@ -0,0 +1,42 @@
+---
+- name: Install Kolide Fleet
+  hosts: "fleet"
+  become: true
+
+  vars_files:
+    - vars/variables.yml
+
+  environment: "{{ deployment_environment_variables | default({}) }}"
+
+  gather_facts: "{{ osa_gather_facts | default(True) }}"
+
+  tasks:
+    # install SSL certs
+    - include_tasks: fleetSSL.yml
+
+    # install software requirements
+    - include_tasks: fleetRequirements.yml
+
+    # install kolide fleet server
+    - include_tasks: fleetServerInstall.yml
+
+    # drop the configuration
+    - include_tasks: fleetConfig.yml
+
+    # add files for systemd
+    - include_tasks:  fleetService.yml
+      when:
+        - ansible_service_mgr == "systemd"
+
+    # migrate the database
+    - include_tasks:  fleetMigrateDB.yml
+      run_once: true
+
+    # start fleet via systemd
+    - include_tasks:  fleetStartService.yml
+
+    # configure kolide fleet & set admin account
+    - include_tasks:  fleetRegisterAdmin.yml
+
+    # retrieve and set enrollment token
+    - include_tasks:  fleetGetEnrollmentToken.yml

osquery/installMariaDB.yml

@@ -0,0 +1,29 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Install MariaDB server
+  hosts: mariadb
+  become: true
+  vars_files:
+    - vars/variables.yml
+
+  environment: "{{ deployment_environment_variables | default({}) }}"
+
+  gather_facts: "{{ osa_gather_facts | default(True) }}"
+  serial: 1
+  user: root
+
+  roles:
+    - role: "mariadb"

osquery/installOsquery.yml

@@ -1,5 +1,5 @@
 ---
-# Copyright 2016, Rackspace US, Inc.
+# Copyright 2018, Rackspace US, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,8 +14,37 @@
 # limitations under the License.
 
 - name: Install osquery
-  hosts: osquery_hosts
+  hosts: hosts
   become: true
+  vars_files:
+    - vars/variables.yml
+
+  environment: "{{ deployment_environment_variables | default({}) }}"
+
+  gather_facts: "{{ osa_gather_facts | default(True) }}"
+
+  pre_tasks:
+    - name: create osquery dir
+      file:
+        path: /etc/osquery/ssl
+        state: directory
+
+    - name: Debug fleet_config
+      debug:
+        var: hostvars[groups['fleet'][0]]['ansible_host']
+        verbosity: 2
+
+    - name: Distribute self signed ssl cert
+      copy:
+        dest: "{{ kolide_fleet_ssl_cert }}"
+        content: "{{ hostvars[groups['fleet'][0]]['kolide_fleet_ssl_cert_fact'] | b64decode }}"
+        mode: "0640"
+
+    - name: write enroll secret
+      copy:
+        dest: "{{ osquery_enroll_secret_dir }}"
+        content: "{{ hostvars[groups['fleet'][0]]['kolide_fleet_enroll_secret'] }}"
+        mode: "0640"
 
   roles:
-    - osquery
+    - role: "osquery"

osquery/inventory.example.yml

@@ -1,5 +1,32 @@
 ---
-# This is the location where osquery(s) will live
-osquery_hosts:
+
+################################## ALL HOSTS ##################################
+all:
+  hosts:
+    # Local host
+    localhost:
+      ansible_connection: local
+
+################################## REQUIRED ###################################
+    logging01:
+      ansible_host: 172.16.27.100
+      ansible_user: root
+
+  vars: {}
+
+
+################################### GROUPS ####################################
+
+# The hosts group is used to target physical host machines. Enter all physical
+# host machines here.
+hosts:
+  hosts:
+    logging01:
+
+# This is the location where fleet(s) will live
+fleet:
+  hosts:
+    logging01:
+osquery:
   hosts:
     all:

osquery/readme.rst

@@ -1,42 +1,187 @@
-Install OSQuery
-###############
+Install OSQuery and Kolide fleet
+################################
 :tags: openstack, ansible
 
+Table of Contents
+=================
+
+      * [About this repository](#about-this-repository)
+      * [OpenStack-Ansible Integration](#openstack-ansible-integration)
+      * [TODO](#todo)
+
+
 About this repository
 ---------------------
 
 This set of playbooks will deploy osquery. If this is being deployed as part of
 an OpenStack all of the inventory needs will be provided for.
 
-There multiple ways to aggregate the data. At this point this repo does not provide
-one of said methods.  It is currently intended to be utilized with the `elk_metrics_6x`.
-
-It is the intention that at a later point to the ability to configure osquery to report
-to a centralized place like (kolide/fleet)[https://github.com/kolide/fleet], (zentral)[https://github.com/zentralopensource/zentral],
-etc.
 
 **These playbooks require Ansible 2.4+.**
 
-Deployment Process
-------------------
+Highlevel overview of Osquery & Kolide Fleet  infrastructure these playbooks will
+build and operate against.
 
-Clone the osa ops repo
+.. image:: assets/place-holder.svg
+    :scale: 50 %
+    :alt: Osquery & Kolide Fleet Architecture Diagram
+    :align: center
+
+OpenStack-Ansible Integration
+-----------------------------
+
+These playbooks can be used as standalone inventory or as an integrated part of
+an OpenStack-Ansible deployment. For a simple example of standalone inventory,
+see ``inventory.example.yml``.
+
+Setup | system configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Clone the osquery-osa repo
 
 .. code-block:: bash
 
     cd /opt
     git clone https://github.com/openstack/openstack-ansible-ops
 
-Clone the osquery role
-
-.. code-block:: bash
-
-    cd /opt
-    git clone https://github.com/devx/ansible-osquery.git /etc/ansible/roles/osquery
-
-install osquery
+Copy the env.d file into place
 
 .. code-block:: bash
 
     cd /opt/openstack-ansible-ops/osquery
-    openstack-ansible installOsquery.yml
+    cp env.d/fleet.yml /etc/openstack_deploy/env.d/
+
+Copy the conf.d file into place
+
+.. code-block:: bash
+
+    cp conf.d/fleet.yml /etc/openstack_deploy/conf.d/
+
+In **fleet.yml**, list your logging hosts under fleet-logstash_hosts to create
+the kolide fleet cluster in multiple containers and one logging host under
+`fleet_hosts` to create the fleet container
+
+.. code-block:: bash
+
+    vi /etc/openstack_deploy/conf.d/fleet.yml
+
+Create the containers
+
+.. code-block:: bash
+
+   cd /opt/openstack-ansible/playbooks
+   openstack-ansible lxc-containers-create.yml -e 'container_group=fleet'
+
+
+Update the `/etc/hosts` file 
+
+.. code-block:: bash
+
+   cd /opt/openstack-ansible/playbooks
+   openstack-ansible openstack-hosts-setup.yml -e 'container_group=fleet'
+
+
+
+Create an haproxy entry for kolide-fleet service 8443
+
+.. code-block:: bash
+
+    cd /opt/openstack-ansible-ops/osquery
+    cat haproxy.example  >> /etc/openstack_deploy/user_variables.yml
+
+    cd /opt/openstack-ansible/playbooks/
+    openstack-ansible haproxy-install.yml --tags=haproxy-service-config
+
+
+Deploying | Installing with embedded Ansible
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If this is being executed on a system that already has Ansible installed but is
+incompatible with these playbooks the script `bootstrap-embedded-ansible.sh` can
+be sourced to grab an embedded version of Ansible prior to executing the
+playbooks.
+
+.. code-block:: bash
+
+    source bootstrap-embedded-ansible.sh
+
+
+Deploying | Manually resolving the dependencies
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This playbook has external role dependencies. If Ansible is not installed with
+the `bootstrap-ansible.sh` script these dependencies can be resolved with the
+``ansible-galaxy`` command and the ``ansible-role-requirements.yml`` file.
+
+* Example galaxy execution
+
+.. code-block:: bash
+
+    ansible-galaxy install -r ansible-role-requirements.yml
+
+
+In the even that some of the modules are alread installed execute the following
+
+.. code-block:: bash
+
+    ansible-galaxy install -r ansible-role-requirements.yml --ignore-errors
+
+
+Once the dependencies are set make sure to set the action plugin path to the
+location of the config_template action directory. This can be done using the
+environment variable `ANSIBLE_ACTION_PLUGINS` or through the use of an
+`ansible.cfg` file.
+
+
+Deploying | The environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Install master/data Fleet nodes on the elastic-logstash containers,
+deploy logstash, deploy Kibana, and then deploy all of the service beats.
+
+.. code-block:: bashG
+
+    cd /opt/openstack-ansible-ops/osquery
+    ansible-playbook site.yml $USER_VARS
+
+
+* The `openstack-ansible` command can be used if the version of ansible on the
+  system is greater than **2.5**. This will automatically pick up the necessary
+  group_vars for hosts in an OSA deployment.
+
+* If required add ``-e@/opt/openstack-ansible/inventory/group_vars/all/all.yml``
+  to import sufficient OSA group variables to define the OpenStack release.
+  Journalbeat will then deploy onto all hosts/containers for releases prior to
+  Rocky, and hosts only for Rocky onwards. If the variable ``openstack_release``
+  is undefined the default behaviour is to deploy Journalbeat to hosts only.
+
+* Alternatively if using the embedded ansible, create a symlink to include all
+  of the OSA group_vars. These are not available by default with the embedded
+  ansible and can be symlinked into the ops repo.
+
+.. code-block:: bash
+
+    ln -s /opt/openstack-ansible/inventory/group_vars /opt/openstack-ansible-ops/osquery/group_vars
+
+
+The individual playbooks found within this repository can be independently run
+at anytime.
+
+Architecture | Data flow
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+This diagram outlines the data flow from within an Elastic-Stack deployment.
+
+.. image:: assets/place-holder.svg
+    :scale: 50 %
+    :alt: Kolide & Osquery Data Flow Diagram
+    :align: center
+
+TODO
+----
+The following is a list of open items.
+ - [ ] Test Redhat familly Operating Systems
+ - [ ] missing mariadb cluster (should all work needs additional vars)
+ - [ ] use haproxy instead of the kolide fleet server ip
+ - [ ] add/update tags
+ - [ ] add testing

osquery/site.yml

@@ -0,0 +1,18 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- import_playbook: installMariaDB.yml
+- import_playbook: installKolideFleet.yml
+- import_playbook: installOsquery.yml

osquery/templates/fleet_config.yml.j2

@@ -0,0 +1,15 @@
+mysql:
+  address: {{ hostvars[groups['mariadb'][0]]['ansible_host'] }}:3306
+  database: {{ kolide_fleet_db_name }}
+  username: {{ kolide_fleet_db_user }}
+  password: {{ kolide_fleet_db_password }}
+redis:
+  address: localhost:6379
+auth:
+  jwt_key: {{ kolide_fleet_jwt_key }}
+server:
+  address: {{ kolide_fleet_address }}
+  cert: {{ kolide_fleet_ssl_cert }}
+  key: {{ kolide_fleet_ssl_key }}
+logging:
+  json: true

osquery/templates/fleet_initd.yml.j2

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+###############
+# SysV Init Information
+# description: docker daemon.
+### BEGIN INIT INFO
+# Provides: me
+# Required-Start:
+# Required-Stop:
+# Default-Start: 2 3 5
+# Default-Stop: 0 1 6
+# Short-Description: manage docker daemon
+# Description: docker daemon
+### END INIT INFO
+
+
+HOME=path/to/code
+EXEC=docker
+NAME="fleet_service"
+
+LABEL={{ kolide_fleet_version }}
+PORT=8412
+
+case "$1" in
+    start)
+        RUNNING=`docker inspect -f '{{.State.Running}}'  ${NAME}`
+
+        if [[ ${RUNNING} == "true" ]]; then
+            #
+            # No need to start it
+            #
+            echo "Process is running"
+            exit 1;
+        if
+        echo "Starting container: '$NAME'..."
+        docker run --restart=always -v /etc/fleet:/etc/fleet --name=fleet_service kolide/fleet:{{ kolide_fleet_version }} -p 8412:8412 --config=/etc/fleet/fleet_config.yml
+        ;;
+    stop)
+        echo "Stopping $NAME..."
+
+        docker stop -t 2 ${NAME} &> /dev/null
+
+        if [[ 0 != $? ]]; then
+            echo "Could not stop container"
+            exit 4;
+        fi
+        ;;
+    status)
+        RUNNING=`docker inspect -f '{{.State.Running}}'  ${NAME}`
+
+        if [[ "true" == ${RUNNING} ]]; then
+            echo "$NAME is running"
+        else
+            echo "$NAME is not running"
+        fi
+        ;;
+    inspect)
+        docker inspect ${NAME}
+        ;;
+    logs)
+        docker logs ${NAME}
+        ;;
+    restart)
+        $0 stop
+        $0 start
+        ;;
+    *)
+        echo "Please use start, stop, restart, inspect or status as first argument"
+        ;;
+esac

osquery/templates/fleet_service.yml.j2

@@ -0,0 +1,17 @@
+[Unit]
+Requires=redis.service
+After=redis.service
+
+[Service]
+TimeoutStartSec=0
+ExecStartPre=-/usr/bin/docker pull kolide/fleet:{{ kolide_fleet_version }}
+ExecStartPre=-/usr/bin/docker kill fleet_service
+ExecStartPre=-/usr/bin/docker rm fleet_service
+ExecStart=/usr/bin/docker run -v /etc/fleet:/etc/fleet --name fleet_service kolide/fleet:{{ kolide_fleet_version }} --config=/etc/fleet/fleet_config.yml
+
+ExecStop=-/usr/bin/docker stop fleet_fleet
+
+ExecReload=/usr/bin/docker restart fleet_service
+
+[Install]
+WantedBy=multi-user.target

osquery/templates/redis_initd.yml.j2

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+###############
+# SysV Init Information
+# description: docker daemon.
+### BEGIN INIT INFO
+# Provides: me
+# Required-Start:
+# Required-Stop:
+# Default-Start: 2 3 5
+# Default-Stop: 0 1 6
+# Short-Description: manage docker daemon
+# Description: docker daemon
+### END INIT INFO
+
+
+HOME=path/to/code
+EXEC=docker
+NAME="redis_service"
+
+LABEL={{ kolide_fleet_version }}
+
+case "$1" in
+    start)
+        RUNNING=`docker inspect -f '{{.State.Running}}'  ${NAME}`
+
+        if [[ ${RUNNING} == "true" ]]; then
+            #
+            # No need to start it
+            #
+            echo "Process is running"
+            exit 1;
+        if
+        echo "Starting container: '$NAME'..."
+        docker run -d --restart=always --name=redis_service redis:4-alpine
+        ;;
+    stop)
+        echo "Stopping $NAME..."
+
+        docker stop -t 2 ${NAME} &> /dev/null
+
+        if [[ 0 != $? ]]; then
+            echo "Could not stop container"
+            exit 4;
+        fi
+        ;;
+    status)
+        RUNNING=`docker inspect -f '{{.State.Running}}'  ${NAME}`
+
+        if [[ "true" == ${RUNNING} ]]; then
+            echo "$NAME is running"
+        else
+            echo "$NAME is not running"
+        fi
+        ;;
+    inspect)
+        docker inspect ${NAME}
+        ;;
+    logs)
+        docker logs ${NAME}
+        ;;
+    restart)
+        $0 stop
+        $0 start
+        ;;
+    *)
+        echo "Please use start, stop, restart, inspect or status as first argument"
+        ;;
+esac

osquery/templates/redis_service.yml.j2

@@ -0,0 +1,18 @@
+[Unit]
+Description=Redis Container
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=simple
+restart=always
+TimeoutStartSec=120
+ExecStartPre=-/usr/bin/docker pull redis:4-alpine
+ExecStartPre=-/usr/bin/docker kill redis_service
+ExecStartPre=-/usr/bin/docker rm redis_service
+ExecStart=/usr/bin/docker run --name redis_service redis:alpine
+
+ExecStop=/usr/bin/docker stop redis_service
+
+[Install]
+WantedBy=multi-user.target

osquery/vars/variables.yml

@@ -0,0 +1,61 @@
+kolide_fleet_enable: true
+kolide_fleet_cluster: false
+
+# Kolide Fleet vars
+kolide_fleet_db_name: fleet
+kolide_fleet_db_user: fleet
+kolide_fleet_db_password: fleetSecrete
+
+kolide_fleet_port: "443"
+kolide_fleet_address: "0.0.0.0:{{ kolide_fleet_port }}"
+kolide_fleet_version: "2.0.0-rc3"
+kolide_fleet_url: "https://github.com/kolide/fleet/releases/download"
+
+kolide_fleet_admin_email: admin@openstack.org
+kolide_fleet_admin_password: AdminSecrete
+
+kolide_fleet_ssl_cert: /etc/ssl/certs/fleet.cert
+kolide_fleet_ssl_key: /etc/ssl/private/fleet.key
+kolide_fleet_ssl_pem: /etc/ssl/private/fleet.pem
+kolide_fleet_ssl_ca_cert: /etc/ssl/certs/fleet-ca.pem
+kolide_fleet_ssl_self_signed_subject: "/C=US/ST=Texas/L=San Antonio/O=IT/CN={{ external_lb_vip_address }}/subjectAltName=IP.1={{ external_lb_vip_address }}/subjectAltName=IP.2={{ ansible_host }}}/subjectAltName=IP.3=localhost"
+
+kolide_fleet_ssl_protocol: "{{ ssl_protocol | default('ALL -SSLv2 -SSLv3') }}"
+kolide_fleet_ssl_cipher_suite: "{{ ssl_cipher_suite | default('ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS') }}"
+
+#kolide_fleet_osquery_enroll_secret: "{{ kolide_fleet_enroll_secret }}"
+
+# Osquery vars
+osquery_enroll_secret_dir: /etc/osquery/osquery_enroll_secret
+
+osquery_flags:
+    - "--tls_server_certs={{ kolide_fleet_ssl_cert }}"
+    - "--tls_hostname={{  hostvars[groups['fleet'][0]]['ansible_host'] }}:{{ kolide_fleet_port }}"
+    - "--host_identifier=hostname"
+    - "--enroll_tls_endpoint=/api/v1/osquery/enroll"
+    - "--config_plugin=tls"
+    - "--config_tls_endpoint=/api/v1/osquery/config"
+    - "--config_tls_refresh=10"
+    - "--disable_distributed=false"
+    - "--distributed_plugin=tls"
+    - "--distributed_interval=10"
+    - "--distributed_tls_max_attempts=3"
+    - "--distributed_tls_read_endpoint=/api/v1/osquery/distributed/read"
+    - "--distributed_tls_write_endpoint=/api/v1/osquery/distributed/write"
+    - "--logger_plugin=tls"
+    - "--logger_tls_endpoint=/api/v1/osquery/log"
+    - "--logger_tls_period=10"
+    - "--enroll_secret_path={{ osquery_enroll_secret_dir }}"
+
+# MariaDB/Gallera Variables
+mariadb_root_password: fleetSecrete
+mariadb_bind_address: "0.0.0.0"
+mariadb_root_remote: 1
+mariadb_databases:
+  - name: "{{ kolide_fleet_db_name }}"
+
+mariadb_users:
+  - name: "{{ kolide_fleet_db_user }}"
+    password: "{{ kolide_fleet_db_password }}"
+    priv: " {{ kolide_fleet_db_name }}.*:ALL"
+    host: "%"