Add ELK to the cloud

This change adds the elk_metrics_6x deployment tooling to this project.
A new submodule was added for the openstack-ansible-ops repo which will
be setup and executed via the deployment script.

Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

.gitmodules

@@ -2,3 +2,6 @@
 	path = openstack-ansible
 	url = https://git.openstack.org/openstack/openstack-ansible
 	branch = master
+[submodule "openstack-ansible-ops"]
+	path = openstack-ansible-ops
+	url = https://git.openstack.org/openstack/openstack-ansible-ops

openstack-ansible-ops

@@ -0,0 +1 @@
+Subproject commit 42f7f896b45dfe3f80969cab980683077dd70120

openstack_deploy/env.d/elk.yml

@@ -0,0 +1,61 @@
+---
+component_skel:
+  apm-server:
+    belongs_to:
+      - elk_all
+      - apm_all
+  elastic-logstash:
+    belongs_to:
+      - elk_all
+      - elasticsearch
+      - elasticsearch_all
+      - logstash
+      - logstash_all
+  kibana:
+    belongs_to:
+      - elk_all
+
+# Note this is all running "is_metal" This is done because the services will run
+# from a dedicated VM.
+container_skel:
+  apm-server_container:
+    belongs_to:
+      - apm-server_containers
+    contains:
+      - apm-server
+    properties:
+      is_metal: true
+  elastic-logstash_container:
+    belongs_to:
+      - elastic-logstash_containers
+    contains:
+      - elastic-logstash
+    properties:
+      is_metal: true
+  kibana_container:
+    belongs_to:
+      - kibana_containers
+    contains:
+      - kibana
+    properties:
+      is_metal: true
+
+physical_skel:
+  apm-server_containers:
+    belongs_to:
+      - all_containers
+  apm-server_hosts:
+    belongs_to:
+      - hosts
+  elastic-logstash_containers:
+    belongs_to:
+      - all_containers
+  elastic-logstash_hosts:
+    belongs_to:
+      - hosts
+  kibana_containers:
+    belongs_to:
+      - all_containers
+  kibana_hosts:
+    belongs_to:
+      - hosts

openstack_deploy/openstack_user_config.yml

@@ -74,6 +74,17 @@ orchestration_hosts: *infrastructure_hosts
 dashboard_hosts: *infrastructure_hosts
 network_hosts: *infrastructure_hosts
 
+# Elastic-Stack
+kibana_hosts: &elk_hosts
+  elk1:
+    ip: 10.4.70.59
+  elk2:
+    ip: 10.4.70.31
+  elk3:
+    ip: 10.4.70.67
+elastic-logstash_hosts: *elk_hosts
+apm-server_hosts: *elk_hosts
+
 # nova hypervisors
 compute_hosts: &compute_hosts
   lsn-d5509:

openstack_deploy/user_variables.yml

@@ -105,3 +105,71 @@ cinder_backends:
     rbd_user: "{{ cinder_ceph_client }}"
     rbd_secret_uuid: "{{ cinder_ceph_client_uuid }}"
     report_discard_supported: true
+
+# uwsgi monitoring overrides
+keystone_uwsgi_ini_overrides:
+  uwsgi:
+    stats: "/tmp/keystone-uwsgi-stats.sock"
+
+cinder_api_uwsgi_ini_overrides:
+  uwsgi:
+    stats: "/tmp/cinder-api-uwsgi-stats.sock"
+
+glance_api_uwsgi_ini_overrides:
+  uwsgi:
+    stats: "/tmp/glance-api-uwsgi-stats.sock"
+
+heat_api_uwsgi_ini_overrides:
+  uwsgi:
+    stats: "/tmp/heat-api-uwsgi-stats.sock"
+
+heat_api_cfn_init_overrides:
+  uwsgi:
+    stats: "/tmp/heat-api-cfn-uwsgi-stats.sock"
+
+nova_api_metadata_uwsgi_ini_overrides:
+  uwsgi:
+    stats: "/tmp/nova-api-metadata-uwsgi-stats.sock"
+
+nova_api_os_compute_uwsgi_ini_overrides:
+  uwsgi:
+    stats: "/tmp/nova-api-os-compute-uwsgi-stats.sock"
+
+nova_placement_uwsgi_ini_overrides:
+  uwsgi:
+    stats: "/tmp/nova-placement-uwsgi-stats.sock"
+
+octavia_api_uwsgi_ini_overrides:
+  uwsgi:
+    stats: "/tmp/octavia-api-uwsgi-stats.sock"
+
+sahara_api_uwsgi_ini_overrides:
+  uwsgi:
+    stats: "/tmp/sahara-api-uwsgi-stats.sock"
+
+ironic_api_uwsgi_ini_overrides:
+  uwsgi:
+    stats: "/tmp/ironic-api-uwsgi-stats.sock"
+
+magnum_api_uwsgi_ini_overrides:
+  uwsgi:
+    stats: "/tmp/magnum-api-uwsgi-stats.sock"
+
+# Adding elastic search and kibana to haproxy_extra_services
+haproxy_extra_services:
+  - service:
+      haproxy_service_name: elasticsearch
+      haproxy_backend_nodes: "{{ groups['elastic-logstash'] | default([]) }}"
+      haproxy_ssl: True
+      haproxy_port: 9201
+      haproxy_backend_port: 9200
+      haproxy_balance_type: http
+      haproxy_backend_options:
+        - "httpchk"
+  - service:
+      haproxy_service_name: kibana_ssl
+      haproxy_backend_nodes: "{{ groups['kibana'] | default([]) }}"
+      haproxy_ssl: True
+      haproxy_port: 8443
+      haproxy_backend_port: 81
+      haproxy_balance_type: tcp

scripts/run_deploy.sh

@@ -19,3 +19,9 @@ popd
 pushd ${PROJECT_PATH}/network_bootstrap
 openstack-ansible bootstrap-neutron.yml
 popd
+
+pushd ${PROJECT_PATH}/openstack-ansible-ops/elk_metrics_6x
+source bootstrap-embedded-ansible.sh
+ansible-playbook site.yml "${USER_VARS}" -e 'elk_package_state="latest"'
+deactivate
+popd

ssh/authorized_keys

@@ -1,2 +1,3 @@
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDZOgw2Qml7bHBr/IyDHmjij7B+/Hu5ysYFEoAW8FMpvnVfv18wdTgYxOfDkGm4Ld5gHQE/mrP44WULFiMgpajMJS0JzB6A8thULKTK/I9j9fibte/pZmT5+TRwI5A0ZLa1tafNzCnKiS2y614GRk7uQlQEEZNk6fDcinP9tLSZjXoqCxbGG1WCg/PMqFPvduwPQDLxDtaDrSsrUq6AzjGescp9ucU3evgWKcROPWCIOYLyQVWrZ9wImUq25qh9Bfc34zYFqbJktv85A0xT/hOePn5itsrDL1MzqJR/jSD9HftaSrhcKJ6KEXoGmQf2yYi6c2ltHCy4adDU0/yuQ2u1CuU4o+xPBkl/3ZmLl+kzKGo1zXAsqZo31Jgrc10tD3F+NQ4EWCg8t9fxG6pYmhSO2GiokFBDBcNMwriN+6pTdKPVdupKeUZBdv8CdFwo60M+QLnT1OtNMz0SPyOUiYCs6QIevEYwgi4pYesoeHMbZ74M9iZmVzFa7QwGEpOQiA6fSffBCnW8FtFA3nr8X7BbgyS5vzeqZwyKwfOIfldGdDm4VKKea06e4mK2i0aX9WyTqsmLSC9WSv1DI4gecYVTih3Dt8uY+1NpY1LRrSIwNp4J71aw9UfWcStRbdWQcMDm9VtiI+2bf459gEGS2ml5N0vHQK5MkZd8j0EWLcEniw== ubuntu@continuouspw-osa
 ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAfgK5Fv5pGOK3eA9WY4EknPF1kWBLYFu4/OVaXMMbL4J9JdQrDjyY8n7iirs48UXtVNJ1n4vVdM4Vs2jGre3iZV/ZZOc+qhK+7usvMO0FKsAUBxh8vMyJhub8Q7JZO4hTZt9B0ID0Sj68f0xHca4zVpQUwsxxs+boUwsy/zVr0Wx7cm7wf8QEJnk8xHxDV6J65CxBmOm6kxWk17IJTInhbLbzFp+N9yeUIwttXV2fIvWrMhBS9U7K47NS7A1ePNdu+ZqMTKAYGgKimxrHUefhWdRbyYtM3CSVu6Vi2nKScnCFDNZI6SCG3BaXmzq4tCovqnje16cI/B2vVRbda6K5O2Gn+YPGc5XiwiSsfJdq4RMOCjOkhn6Fmei5mS7EQFXZh1ZxFWwNSpyOp0xt8B1+8YDe1QRiX3WYq5YAgeulhljwENCrBLjR77juJuxfHCTxJunY3EHPYFOKt9Zw/BKumUaSQRojAMUkumWG4Y+LcAV8IwmkbEU7IlLkUYe+1j8zH66KWqTfHPKM9naWaWq5UMpympRBRmgb8kSqfir0f/oUfZmQsFMqcI8MxrYLQQkTUKbVjTa8ZuCMxXLePIrbj4Lk/O2VYyWQ1Wnj610DpQCZFA97CG30GKPIyr7A7uraVR6/vdd1dsJLknPGPG4IcTpkD3xfqO8Qv8= logan
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDS5UeOQxwoW93yT/nBE+Y0/vsiBmhPjBOruggXS9IqlCF8ut6wEBEc1ne2hDqfLZZUWg4rorZPAmp5C5tco/L4jCb7/xnjrNBN5x8h6sYJyhz5dIuZBWKCM4Yq2lSNHnuTufZoxmz0sbT3skOU9WArTLjkqsV2r8fgixeCvDirjDa4AHPYp3N9VM+69NHoIseYUkJhFCpDoztO/gYZsJcx9p+gfUlqQJ75m6G+DWAkyRtImkA3b2uykKpnurEquk7Hub8PxgkLj6pYsL4jfSY7rnqmdQqjiLXNj0PR+rKOExe58J0ho90yPhrRy3UfHCh33E7at2Na25TdKLZhpbc3 cloudnull@kevins-mbp