Merge "Support bypassing Monasca Log API for control plane logs"

ansible/group_vars/all.yml

@@ -711,6 +711,11 @@ enable_destroy_images: "no"
 ####################
 monasca_enable_alerting_pipeline: True
 
+# Send logs from the control plane to the Monasca API. Monasca will then persist
+# them in Elasticsearch. If this is disabled, control plane logs will be sent
+# directly to Elasticsearch.
+monasca_ingest_control_plane_logs: True
+
 monasca_api_admin_base_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn  | put_address_in_context('url') }}:{{ monasca_api_port }}"
 monasca_api_internal_base_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn  | put_address_in_context('url') }}:{{ monasca_api_port }}"
 monasca_api_public_base_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn  | put_address_in_context('url') }}:{{ monasca_api_port }}"

ansible/roles/common/tasks/config.yml

@@ -106,7 +106,7 @@
     log_direct_to_elasticsearch: >-
       {{ ( enable_elasticsearch | bool or
            ( elasticsearch_address != kolla_internal_vip_address )) and
-         not enable_monasca | bool }}
+           ( not enable_monasca | bool or not monasca_ingest_control_plane_logs | bool ) }}
     fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
     # Inputs
     fluentd_input_files: "{{ default_input_files | customise_fluentd(customised_input_files) }}"
@@ -143,7 +143,7 @@
       - name: "conf/output/01-es.conf.j2"
         enabled: "{{ log_direct_to_elasticsearch }}"
       - name: "conf/output/02-monasca.conf.j2"
-        enabled: "{{ enable_monasca | bool }}"
+        enabled: "{{ enable_monasca | bool and monasca_ingest_control_plane_logs | bool }}"
     customised_output_files: "{{ find_custom_fluentd_outputs.files | map(attribute='path') | list }}"
   template:
     src: "td-agent.conf.j2"

ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2

@@ -71,7 +71,7 @@
     </record>
 </filter>
 
-{% if enable_monasca | bool %}
+{% if enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
 # Kolla configures Fluentd to extract timestamps from OpenStack service
 # logs, however these timestamps are not saved in the event and are not
 # forwarded to Monasca. Here we save the timestamp which has been

ansible/roles/common/templates/conf/output/00-local.conf.j2

@@ -36,7 +36,7 @@
        buffer_type file
        buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_swift_facility }}.*
   </store>
-{% elif enable_monasca | bool %}
+{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
   <store>
      @type monasca
      keystone_url {{ keystone_internal_url }}
@@ -98,7 +98,7 @@
        buffer_type file
        buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_haproxy_facility }}.*
   </store>
-{% elif enable_monasca | bool %}
+{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
   <store>
      @type monasca
      keystone_url {{ keystone_internal_url }}
@@ -158,7 +158,7 @@
        buffer_type file
        buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_glance_tls_proxy_facility }}.*
   </store>
-{% elif enable_monasca | bool %}
+{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
   <store>
      @type monasca
      keystone_url {{ keystone_internal_url }}
@@ -219,7 +219,7 @@
        buffer_type file
        buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_neutron_tls_proxy_facility }}.*
   </store>
-{% elif enable_monasca | bool %}
+{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
   <store>
      @type monasca
      keystone_url {{ keystone_internal_url }}

ansible/roles/elasticsearch/defaults/main.yml

@@ -59,7 +59,7 @@ elasticsearch_curator_dry_run: false
 
 # Index prefix pattern. Any indices matching this regex will
 # be managed by Curator.
-elasticsearch_curator_index_pattern: "^{{ 'monasca' if enable_monasca|bool else kibana_log_prefix }}-.*"
+elasticsearch_curator_index_pattern: "^{{ '(monasca|' + kibana_log_prefix + ')' if enable_monasca|bool else kibana_log_prefix }}-.*"
 
 # Duration after which an index is staged for deletion. This is
 # implemented by closing the index. Whilst in this state the index

doc/source/reference/logging-and-monitoring/monasca-guide.rst

@@ -49,6 +49,33 @@ resource usage you can set ``/etc/kolla/globals.yml``:
 
    monasca_enable_alerting_pipeline: "no"
 
+You can optionally bypass Monasca for control plane logs, and instead have
+them sent directly to Elasticsearch. This should be avoided if you have
+deployed Monasca as a standalone service for the purpose of storing
+logs in a protected silo for security purposes. However, if this is not
+a relevant consideration, for example you have deployed Monasca alongside the
+existing Openstack control plane, then you may free up some resources by
+setting:
+
+.. code-block:: yaml
+
+   monasca_ingest_control_plane_logs: "no"
+
+You should note that when making this change with the default
+``kibana_log_prefix`` prefix of ``flog-``, you will need to create a new
+index pattern in Kibana accordingly. If you wish to continue to search all
+logs using the same index pattern in Kibana, then you can override
+``kibana_log_prefix`` to ``monasca`` or similar in ``/etc/kolla/globals.yml``:
+
+.. code-block:: yaml
+
+   kibana_log_prefix: "monasca"
+
+If you have enabled Elasticsearch Curator, it will be configured to rotate
+logs with index patterns matching either ``^flog-.*`` or ``^monasca-.*`` by
+default. If this is undesirable then you can update the
+``elasticsearch_curator_index_pattern`` variable accordingly.
+
 Currently Monasca is only supported using the ``source`` install type Kolla
 images. If you are using the ``binary`` install type you should set the
 following override in ``/etc/kolla/globals.yml``:

releasenotes/notes/support-bypassing-monasca-for-control-plane-logs-651a47edf440808b.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Optionally support sending control plane logs directly to
+    Elasticsearch when Monasca is enabled.