openstack/openstack-ansible-ops
Code Issues Proposed changes

Deploy ELK in MNAIO

Browse Source 
This enables the deployment of the elk_metrics_6x stack inside of an
MNAIO.

Change-Id: Ie611baee79c33d7cbab9f0865127ac5966475838
This commit is contained in:
Dave Wilde 2018-07-27 11:21:37 -05:00 committed by David Wilde
parent 86a2402da9
commit 525887872b
11 changed files with 305 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
elk_metrics_6x/roles/elastic_curator/tasks/main.yml
View File

@ -38,6 +38,11 @@
tags:
- package_install
- name: Create the virtualenv (if it does not exist)
command: "virtualenv --never-download --no-site-packages /opt/elasticsearch-curator"
args:
creates: "/opt/elasticsearch-curator/bin/activate"
- name: Ensure curator is installed
pip:
name: "elasticsearch-curator<6"

13
multi-node-aio/README.rst
View File

@ -117,7 +117,6 @@ Instruct the system do all of the required PXE setup:
Instruct the system do all of the required DHCPD setup:
``SETUP_DHCPD=${SETUP_DHCPD:-true}``
Instruct the system to Kick all of the VMs:
``DEPLOY_VMS=${DEPLOY_VMS:-true}``
@ -130,12 +129,21 @@ Instruct the VM to use the selected kernel meta package, eg. linux-generic:
Set the OSA repo for this script to retrieve:
``OSA_REPO=${OSA_REPO:-https://git.openstack.org/openstack/openstack-ansible}``
Set the openstack-ansible-ops repo to retrieve for the ELK stack:
``OS_OPS_REPO=${OS_OPS_REPO:-https://git.openstack.org/openstack/openstack-ansible-ops}``
Set the OSA branch for this script to deploy:
``OSA_BRANCH=${OSA_BRANCH:-master}``
Set the openstack-ansible-ops branch for this script to deploy:
``OS_OPS_BRANCH=${OS_OPS_BRANCH:-master}``
Instruct the system to deploy OpenStack Ansible:
``DEPLOY_OSA=${DEPLOY_OSA:-true}``
Instruct the system to deploy the ELK Stack:
``DEPLOY_ELK=${DEPLOY_ELK:-false}``
Instruct the system to pre-config the envs for running OSA playbooks:
``PRE_CONFIG_OSA=${PRE_CONFIG_OSA:-true}``
@ -143,6 +151,9 @@ Instruct the system to run the OSA playbooks, if you want to deploy other OSA
powered cloud, you can set it to false:
``RUN_OSA=${RUN_OSA:-true}``
Instruct the system to run the ELK playbooks:
``RUN_ELK=${RUN_ELK:-false}``
Instruct the system to configure the completed OpenStack deployment with some
example flavors, images, networks, etc.:
``CONFIGURE_OPENSTACK=${CONFIGURE_OPENSTACK:-true}``

4
multi-node-aio/build.sh
View File

@ -33,13 +33,17 @@ ansible-playbook -vv \
-e setup_dhcpd=${SETUP_DHCPD:-"true"} \
-e deploy_vms=${DEPLOY_VMS:-"true"} \
-e deploy_osa=${DEPLOY_OSA:-"true"} \
-e deploy_elk=${DEPLOY_ELK:-"false"} \
-e osa_repo=${OSA_REPO:-"https://git.openstack.org/openstack/openstack-ansible"} \
-e os_ops_repo=${OS_OPS_REPO:-"https://git.openstack.org/openstack/openstack-ansible-ops"} \
-e osa_branch=${OSA_BRANCH:-"master"} \
-e os_ops_branch=${OS_OPS_BRANCH:-"master"} \
-e default_network=${DEFAULT_NETWORK:-"eth0"} \
-e default_image=${DEFAULT_IMAGE:-"ubuntu-16.04-amd64"} \
-e vm_disk_size=${VM_DISK_SIZE:-92160} \
-e http_proxy=${http_proxy:-''} \
-e run_osa=${RUN_OSA:-"true"} \
-e run_elk=${RUN_ELK:-"false"} \
-e pre_config_osa=${PRE_CONFIG_OSA:-"true"} \
-e configure_openstack=${CONFIGURE_OPENSTACK:-"true"} \
-e config_prerouting=${CONFIG_PREROUTING:-"false"} \

64
multi-node-aio/playbooks/deploy-elk.yml Normal file
View File

@ -0,0 +1,64 @@
---
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in witing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Setup deploy host
hosts: deploy_hosts
gather_facts: "{{ gather_facts | default(true) }}"
environment: "{{ deployment_environment_variables | default({}) }}"
tags:
- deploy-osa
tasks:
- name: Gather variables for each operating system
include_vars: "{{ item }}"
with_first_found:
- "{{ playbook_dir }}/vars/{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
- "{{ playbook_dir }}/vars/{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ playbook_dir }}/vars/{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ playbook_dir }}/vars/{{ ansible_distribution | lower }}.yml"
- "{{ playbook_dir }}/vars/{{ ansible_os_family | lower }}.yml"
tags:
- always
# Example read-write git checkout from github
- name: Get openstack-ansible-ops
git:
repo: "{{ os_ops_repo | default('https://git.openstack.org/openstack/openstack-ansible-ops') }}"
dest: /opt/openstack-ansible-ops
version: "{{ os_ops_branch | default('master') }}"
force: true
when:
- deploy_elk | default(false) | bool
- name: Run ELK
command: "{{ item }}"
args:
chdir: /opt/openstack-ansible-ops/elk_metrics_6x
with_items:
- tmux attach -t build-osa
- tmux select-pane -t 0
- tmux send-keys "openstack-ansible playbooks/lxc-containers-create --limit elk_all" C-m
- tmux send-keys "cd /opt/openstack-ansible-ops/elk_metrics_6x" C-m
- tmux send-keys "openstack-ansible site.yml" C-m
when:
- inventory_hostname == groups['deploy_hosts'][0]
- run_elk | default(false) | bool
- name: Finished notice
debug:
msg: |
ELK deploy running. To check on the state of this deployment, login
to the {{ groups['deploy_hosts'][0] }} VM and attach to the "build-osa" tmux session.
when:
- run_elk | default(false) | bool

22
multi-node-aio/playbooks/deploy-osa.yml
View File

@ -98,6 +98,28 @@
- pre_config_osa | default(true) | bool
- osa_disable_serial | default(false) | bool
- name: Drop ELK env.d config
template:
src: "osa/elk-envd.yml"
dest: "/etc/openstack_deploy/env.d/elk.yml"
mode: "0644"
owner: root
group: root
when:
- pre_config_osa | default(true) | bool
- osa_enable_elk_metrics | default(false) | bool
- name: Drop ELK conf.d config
template:
src: "osa/elk-confd.yml"
dest: "/etc/openstack_deploy/conf.d/elk.yml"
mode: "0644"
owner: root
group: root
when:
- pre_config_osa | default(true) | bool
- osa_enable_elk_metrics | default(false) | bool
- name: Ensure the user_variables file is populated
lineinfile:
path: /etc/openstack_deploy/user_variables.yml

3
multi-node-aio/playbooks/group_vars/all.yml
View File

@ -148,3 +148,6 @@ osa_enable_network: true
osa_enable_meter: true
osa_enable_object_storage: true
osa_disable_serial: false
osa_enable_elk_metrics: false
osa_enable_os_profiler: false
osa_enable_uwsgi_stats: false

19
multi-node-aio/playbooks/osa/elk-confd.yml Normal file
View File

@ -0,0 +1,19 @@
# For the puposes of this example, the kibana nodes have been added to
# different host machines that the logging nodes. The intention here
# is to show that the different components can scale independently of
# one another.
kibana_hosts:
infra1:
ip: 10.0.236.100
infra2:
ip: 10.0.236.101
infra3:
ip: 10.0.236.102
elastic-logstash_hosts:
logging1:
ip: 10.0.236.110
apm-server_hosts:
logging1:
ip: 10.0.236.110

53
multi-node-aio/playbooks/osa/elk-envd.yml Normal file
View File

@ -0,0 +1,53 @@
---
component_skel:
apm-server:
belongs_to:
- elk_all
- apm_all
elastic-logstash:
belongs_to:
- elk_all
- elasticsearch
- elasticsearch_all
- logstash
- logstash_all
kibana:
belongs_to:
- elk_all
container_skel:
apm-server_container:
belongs_to:
- apm-server_containers
contains:
- apm-server
elastic-logstash_container:
belongs_to:
- elastic-logstash_containers
contains:
- elastic-logstash
kibana_container:
belongs_to:
- kibana_containers
contains:
- kibana
physical_skel:
apm-server_containers:
belongs_to:
- all_containers
apm-server_hosts:
belongs_to:
- hosts
elastic-logstash_containers:
belongs_to:
- all_containers
elastic-logstash_hosts:
belongs_to:
- hosts
kibana_containers:
belongs_to:
- all_containers
kibana_hosts:
belongs_to:
- hosts

11
multi-node-aio/playbooks/osa/openstack_user_config.yml
View File

@ -235,3 +235,14 @@ swift_hosts: *swift_block
# swift infra hosts
swift-proxy_hosts: *infra_block
{% endif %}
{% if osa_enable_elk_metrics | bool %}
# kibana hosts
kibana_hosts: *infra_block
# elasticsearch/logstash hosts
elastic-logstash_hosts: *log_block
# apm hosts
apm-server_hosts: *log_block
{% endif %}

108
multi-node-aio/playbooks/osa/user_mnaio_variables.yml
View File

@ -44,3 +44,111 @@ glance_glance_api_conf_overrides:
# Reduce memory footprint for mnaio
galera_innodb_buffer_pool_size: 1024M
galera_gcache_size: 128M
{% if osa_enable_elk_metrics | bool %}
haproxy_extra_services:
- service:
haproxy_service_name: elastic-logstash
haproxy_ssl: False
haproxy_backend_nodes: "{{ '{{' }} groups['Kibana'] | default([]) {{ '}}' }}" # Kibana nodes are also Elasticsearch coordination nodes
haproxy_port: 9201 # This is set using the "elastic_hap_port" variable
haproxy_check_port: 9200 # This is set using the "elastic_port" variable
haproxy_backend_port: 9200 # This is set using the "elastic_port" variable
haproxy_balance_type: tcp
- service:
haproxy_service_name: Kibana
haproxy_ssl: False
haproxy_backend_nodes: "{{ '{{' }} groups['Kibana'] | default([]) {{ '}}' }}"
haproxy_port: 81 # This is set using the "Kibana_nginx_port" variable
haproxy_balance_type: tcp
- service:
haproxy_service_name: apm-server
haproxy_ssl: False
haproxy_backend_nodes: "{{ '{{' }} groups['apm-server'] | default([]) {{ '}}' }}"
haproxy_port: 8200 # this is set using the "apm_port" variable
haproxy_balance_type: tcp
{% endif %}
{% if osa_enable_os_profiler | bool %}
profiler_overrides: &os_profiler
profiler:
enabled: true
trace_sqlalchemy: true
hmac_keys: '{{ '{{' }} os_profiler_hmac_token {{ '}}' }}'
connection_string: "Elasticsearch://'{{ '{{' }} internal_lb_vip_address {{ '}}' }}':9201"
es_doc_type: "notification"
es_scroll_time: "2m"
es_scroll_size: "10000"
filter_error_trace: "false"
aodh_aodh_conf_overrides: *os_profiler
barbican_config_overrides: *os_profiler
ceilometer_ceilometer_conf_overrides: *os_profiler
cinder_cinder_conf_overrides: *os_profiler
designate_designate_conf_overrides: *os_profiler
glance_glance_api_conf_overrides: *os_profiler
gnocchi_conf_overrides: *os_profiler
heat_heat_conf_overrides: *os_profiler
horizon_config_overrides: *os_profiler
ironic_ironic_conf_overrides: *os_profiler
keystone_keystone_conf_overrides: *os_profiler
magnum_config_overrides: *os_profiler
neutron_neutron_conf_overrides: *os_profiler
nova_nova_conf_overrides: *os_profiler
octavia_octavia_conf_overrides: *os_profiler
rally_config_overrides: *os_profiler
sahara_conf_overrides: *os_profiler
swift_swift_conf_overrides: *os_profiler
tacker_tacker_conf_overrides: *os_profiler
trove_config_overrides: *os_profiler
{% endif %}
{% if osa_enable_uwsgi_stats | bool %}
keystone_uwsgi_ini_overrides:
uwsgi:
stats: "/tmp/keystone-uwsgi-stats.sock"
cinder_api_uwsgi_ini_overrides:
uwsgi:
stats: "/tmp/cinder-api-uwsgi-stats.sock"
glance_api_uwsgi_ini_overrides:
uwsgi:
stats: "/tmp/glance-api-uwsgi-stats.sock"
heat_api_uwsgi_ini_overrides:
uwsgi:
stats: "/tmp/heat-api-uwsgi-stats.sock"
heat_api_cfn_init_overrides:
uwsgi:
stats: "/tmp/heat-api-cfn-uwsgi-stats.sock"
nova_api_metadata_uwsgi_ini_overrides:
uwsgi:
stats: "/tmp/nova-api-metadata-uwsgi-stats.sock"
nova_api_os_compute_uwsgi_ini_overrides:
uwsgi:
stats: "/tmp/nova-api-os-compute-uwsgi-stats.sock"
nova_placement_uwsgi_ini_overrides:
uwsgi:
stats: "/tmp/nova-placement-uwsgi-stats.sock"
octavia_api_uwsgi_ini_overrides:
uwsgi:
stats: "/tmp/octavia-api-uwsgi-stats.sock"
sahara_api_uwsgi_ini_overrides:
uwsgi:
stats: "/tmp/sahara-api-uwsgi-stats.sock"
ironic_api_uwsgi_ini_overrides:
uwsgi:
stats: "/tmp/ironic-api-uwsgi-stats.sock"
magnum_api_uwsgi_ini_overrides:
uwsgi:
stats: "/tmp/magnum-api-uwsgi-stats.sock"
{% endif %}

4
multi-node-aio/playbooks/site.yml
View File

@ -37,6 +37,10 @@
when:
- deploy_osa | default(true) | bool
- import_playbook: deploy-elk.yml
when:
- deploy_elk | default(false) | bool
- import_playbook: openstack-service-setup.yml
when:
- configure_openstack | default(true) | bool