diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index ce1a41b201..09b77006e6 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -709,6 +709,11 @@ enable_destroy_images: "no" #################### monasca_enable_alerting_pipeline: True +# Send logs from the control plane to the Monasca API. Monasca will then persist +# them in Elasticsearch. If this is disabled, control plane logs will be sent +# directly to Elasticsearch. +monasca_ingest_control_plane_logs: True + monasca_api_admin_base_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_api_port }}" monasca_api_internal_base_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_api_port }}" monasca_api_public_base_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn | put_address_in_context('url') }}:{{ monasca_api_port }}" diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml index d5071b1899..c429ad0d00 100644 --- a/ansible/roles/common/tasks/config.yml +++ b/ansible/roles/common/tasks/config.yml @@ -106,7 +106,7 @@ log_direct_to_elasticsearch: >- {{ ( enable_elasticsearch | bool or ( elasticsearch_address != kolla_internal_vip_address )) and - not enable_monasca | bool }} + ( not enable_monasca | bool or not monasca_ingest_control_plane_logs | bool ) }} fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}" # Inputs fluentd_input_files: "{{ default_input_files | customise_fluentd(customised_input_files) }}" @@ -143,7 +143,7 @@ - name: "conf/output/01-es.conf.j2" enabled: "{{ log_direct_to_elasticsearch }}" - name: "conf/output/02-monasca.conf.j2" - enabled: "{{ enable_monasca | bool }}" + enabled: "{{ enable_monasca | bool and monasca_ingest_control_plane_logs | bool }}" customised_output_files: "{{ find_custom_fluentd_outputs.files | map(attribute='path') | list }}" template: src: "td-agent.conf.j2" diff --git a/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 index 2ffb34bf56..2d5fef5bbd 100644 --- a/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 +++ b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 @@ -71,7 +71,7 @@ -{% if enable_monasca | bool %} +{% if enable_monasca | bool and monasca_ingest_control_plane_logs | bool %} # Kolla configures Fluentd to extract timestamps from OpenStack service # logs, however these timestamps are not saved in the event and are not # forwarded to Monasca. Here we save the timestamp which has been diff --git a/ansible/roles/common/templates/conf/output/00-local.conf.j2 b/ansible/roles/common/templates/conf/output/00-local.conf.j2 index 7d2cd65414..8d4afed43f 100644 --- a/ansible/roles/common/templates/conf/output/00-local.conf.j2 +++ b/ansible/roles/common/templates/conf/output/00-local.conf.j2 @@ -36,7 +36,7 @@ buffer_type file buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_swift_facility }}.* -{% elif enable_monasca | bool %} +{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %} @type monasca keystone_url {{ keystone_internal_url }} @@ -98,7 +98,7 @@ buffer_type file buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_haproxy_facility }}.* -{% elif enable_monasca | bool %} +{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %} @type monasca keystone_url {{ keystone_internal_url }} @@ -158,7 +158,7 @@ buffer_type file buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_glance_tls_proxy_facility }}.* -{% elif enable_monasca | bool %} +{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %} @type monasca keystone_url {{ keystone_internal_url }} @@ -219,7 +219,7 @@ buffer_type file buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_neutron_tls_proxy_facility }}.* -{% elif enable_monasca | bool %} +{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %} @type monasca keystone_url {{ keystone_internal_url }} diff --git a/ansible/roles/elasticsearch/defaults/main.yml b/ansible/roles/elasticsearch/defaults/main.yml index 18a9b15554..c212bfc576 100644 --- a/ansible/roles/elasticsearch/defaults/main.yml +++ b/ansible/roles/elasticsearch/defaults/main.yml @@ -59,7 +59,7 @@ elasticsearch_curator_dry_run: false # Index prefix pattern. Any indices matching this regex will # be managed by Curator. -elasticsearch_curator_index_pattern: "^{{ 'monasca' if enable_monasca|bool else kibana_log_prefix }}-.*" +elasticsearch_curator_index_pattern: "^{{ '(monasca|' + kibana_log_prefix + ')' if enable_monasca|bool else kibana_log_prefix }}-.*" # Duration after which an index is staged for deletion. This is # implemented by closing the index. Whilst in this state the index diff --git a/doc/source/reference/logging-and-monitoring/monasca-guide.rst b/doc/source/reference/logging-and-monitoring/monasca-guide.rst index 947265011c..ecf8d035e9 100644 --- a/doc/source/reference/logging-and-monitoring/monasca-guide.rst +++ b/doc/source/reference/logging-and-monitoring/monasca-guide.rst @@ -49,6 +49,33 @@ resource usage you can set ``/etc/kolla/globals.yml``: monasca_enable_alerting_pipeline: "no" +You can optionally bypass Monasca for control plane logs, and instead have +them sent directly to Elasticsearch. This should be avoided if you have +deployed Monasca as a standalone service for the purpose of storing +logs in a protected silo for security purposes. However, if this is not +a relevant consideration, for example you have deployed Monasca alongside the +existing Openstack control plane, then you may free up some resources by +setting: + +.. code-block:: yaml + + monasca_ingest_control_plane_logs: "no" + +You should note that when making this change with the default +``kibana_log_prefix`` prefix of ``flog-``, you will need to create a new +index pattern in Kibana accordingly. If you wish to continue to search all +logs using the same index pattern in Kibana, then you can override +``kibana_log_prefix`` to ``monasca`` or similar in ``/etc/kolla/globals.yml``: + +.. code-block:: yaml + + kibana_log_prefix: "monasca" + +If you have enabled Elasticsearch Curator, it will be configured to rotate +logs with index patterns matching either ``^flog-.*`` or ``^monasca-.*`` by +default. If this is undesirable then you can update the +``elasticsearch_curator_index_pattern`` variable accordingly. + Currently Monasca is only supported using the ``source`` install type Kolla images. If you are using the ``binary`` install type you should set the following override in ``/etc/kolla/globals.yml``: diff --git a/releasenotes/notes/support-bypassing-monasca-for-control-plane-logs-651a47edf440808b.yaml b/releasenotes/notes/support-bypassing-monasca-for-control-plane-logs-651a47edf440808b.yaml new file mode 100644 index 0000000000..924a40c1f9 --- /dev/null +++ b/releasenotes/notes/support-bypassing-monasca-for-control-plane-logs-651a47edf440808b.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Optionally support sending control plane logs directly to + Elasticsearch when Monasca is enabled.