Remove Monasca Log Transformer

Historically Monasca Log Transformer has been for log
standardisation and processing. For example, logs from different
sources may use slightly different error levels such as WARN, 5,
or WARNING. Monasca Log Transformer is a place where these could
be 'squashed' into a single error level to simplify log searches
based on labels such as these.

However, in Kolla Ansible, we do this processing in Fluentd so
that the simpler Fluentd -> Elastic -> Kibana pipeline also
benefits. This helps to avoid spreading out log parsing
configuration over many services, with the Fluentd Monasca output
plugin being yet another potential place for processing (which
should be avoided). It therefore makes sense to remove this
service entirely, and squash any existing configuration which
can't be moved to Fluentd into the Log Perister service. I.e.
by removing this pipeline, we don't loose any functionality,
we encourage log processing to take place in Fluentd, or at least
outside of Monasca, and we make significant gains in efficiency
by removing a topic from Kafka which contains a copy of all logs
in transit.

Finally, users forwarding logs from outside the control plane,
eg. from tenant instances, should be encouraged to process the
logs at the point of sending using whichever framework they are
forwarding them with. This makes sense, because all Logstash
configuration in Monasca is only accessible by control plane
admins. A user can't typically do any processing inside Monasca,
with or without this change.

Change-Id: I65c76d0d1cd488725e4233b7e75a11d03866095c

ansible/monasca_cleanup.yml

@@ -0,0 +1,14 @@
+---
+- name: Cleanup unused Monasca services
+  hosts:
+    - monasca-api
+    - monasca-grafana
+    - monasca-log-transformer
+    - monasca-log-persister
+    - monasca-log-metrics
+    - monasca-thresh
+    - monasca-notification
+    - monasca-persister
+  roles:
+    - { role: monasca,
+        tags: monasca }

ansible/roles/monasca/defaults/main.yml

@@ -20,10 +20,12 @@ monasca_services:
         mode: "http"
         external: true
         port: "{{ monasca_api_port }}"
+  # NOTE(dszumski): We can remove log_transformer and all other references after the
+  # Wallaby release
   monasca-log-transformer:
     container_name: monasca_log_transformer
     group: monasca-log-transformer
-    enabled: true
+    enabled: false
     image: "{{ monasca_logstash_image_full }}"
     volumes: "{{ monasca_log_transformer_default_volumes + monasca_log_transformer_extra_volumes }}"
     dimensions: "{{ monasca_log_transformer_dimensions }}"
@@ -175,7 +177,7 @@ monasca_all_topics:
   - name: "{{ monasca_transformed_logs_topic }}"
     partitions: "{{ monasca_default_topic_partitions }}"
     replication_factor: "{{ monasca_default_topic_replication_factor }}"
-    enabled: True
+    enabled: False
   - name: "{{ monasca_events_topic }}"
     partitions: "{{ monasca_default_topic_partitions }}"
     replication_factor: "{{ monasca_default_topic_replication_factor }}"

ansible/roles/monasca/handlers/main.yml

@@ -14,21 +14,6 @@
   when:
     - kolla_action != "config"
 
-- name: Restart monasca-log-transformer container
-  vars:
-    service_name: "monasca-log-transformer"
-    service: "{{ monasca_services[service_name] }}"
-  become: true
-  kolla_docker:
-    action: "recreate_or_restart_container"
-    common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
-    image: "{{ service.image }}"
-    volumes: "{{ service.volumes }}"
-    dimensions: "{{ service.dimensions }}"
-  when:
-    - kolla_action != "config"
-
 - name: Restart monasca-log-persister container
   vars:
     service_name: "monasca-log-persister"

ansible/roles/monasca/tasks/cleanup.yml

@@ -0,0 +1,22 @@
+---
+- name: Stop and remove containers for disabled monasca services
+  become: true
+  kolla_docker:
+    action: "stop_and_remove_container"
+    name: "{{ item.value.container_name }}"
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - not item.value.enabled | bool
+  with_dict: "{{ monasca_services }}"
+
+- name: Removing config for any disabled services
+  file:
+    path: "{{ node_config_directory }}/{{ item.key }}"
+    state: "absent"
+  become: true
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - not item.value.enabled | bool
+  with_dict: "{{ monasca_services }}"
+
+# NOTE(dszumski): Docker volume removal is currently a manual procedure

ansible/roles/monasca/tasks/config.yml

@@ -160,29 +160,11 @@
   notify:
     - Restart monasca-api container
 
-- name: Copying over monasca-log-transformer config
-  vars:
-    service: "{{ monasca_services['monasca-log-transformer'] }}"
-  template:
-    src: "{{ item }}"
-    dest: "{{ node_config_directory }}/monasca-log-transformer/log-transformer.conf"
-    mode: "0660"
-  become: true
-  with_first_found:
-    - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-transformer.conf"
-    - "{{ node_custom_config }}/monasca/log-transformer.conf"
-    - "{{ role_path }}/templates/monasca-log-transformer/log-transformer.conf.j2"
-  when:
-    - inventory_hostname in groups[service['group']]
-    - service.enabled | bool
-  notify:
-    - Restart monasca-log-transformer container
-
 - name: Ensuring logstash patterns folder exists
   vars:
-    service: "{{ monasca_services['monasca-log-transformer'] }}"
+    service: "{{ monasca_services['monasca-log-persister'] }}"
   file:
-    path: "{{ node_config_directory }}/monasca-log-transformer/logstash_patterns"
+    path: "{{ node_config_directory }}/monasca-log-persister/logstash_patterns"
     state: "directory"
     mode: "0770"
   become: true
@@ -200,10 +182,10 @@
 
 - name: Copying over custom logstash patterns
   vars:
-    service: "{{ monasca_services['monasca-log-transformer'] }}"
+    service: "{{ monasca_services['monasca-log-persister'] }}"
   template:
     src: "{{ item.path }}"
-    dest: "{{ node_config_directory }}/monasca-log-transformer/logstash_patterns/{{ item.path | basename }}"
+    dest: "{{ node_config_directory }}/monasca-log-persister/logstash_patterns/{{ item.path | basename }}"
     mode: "0660"
   with_items: "{{ monasca_custom_logstash_patterns.files }}"
   become: true
@@ -211,7 +193,7 @@
     - inventory_hostname in groups[service['group']]
     - service.enabled | bool
   notify:
-    - Restart monasca-log-transformer container
+    - Restart monasca-log-persister container
 
 - name: Copying over monasca-log-persister config
   vars:

ansible/roles/monasca/tasks/upgrade.yml

@@ -14,6 +14,8 @@
 
 - import_tasks: config.yml
 
+- import_tasks: cleanup.yml
+
 - import_tasks: check-containers.yml
 
 # NOTE(dszumski): We don't want old Grafana instances running after

ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2

@@ -10,7 +10,7 @@
 input {
     kafka {
         bootstrap_servers => "{{ monasca_kafka_servers }}"
-        topics => ["{{ monasca_transformed_logs_topic }}"]
+        topics => ["{{ monasca_raw_logs_topic }}"]
         group_id => "log_metrics"
         consumer_threads => "{{ monasca_log_pipeline_threads }}"
         codec => json

ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2

@@ -1,15 +1,41 @@
-# Persist transformed logs to Elasticsearch
+# Persist logs to Elasticsearch.
 
 input {
     kafka {
         bootstrap_servers => "{{ monasca_kafka_servers }}"
-        topics => ["{{ monasca_transformed_logs_topic }}"]
+        topics => ["{{ monasca_raw_logs_topic }}"]
         group_id => "log_persister"
         consumer_threads => "{{ monasca_log_pipeline_threads }}"
         codec => json
     }
 }
 
+filter {
+    # Update the timestamp of the event based on the time in the message.
+    date {
+        match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss Z", "ISO8601"]
+        remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ]
+    }
+
+    # Monasca Log API adds a timestamp when it processes a log entry. This
+    # timestamp needs to be converted from seconds since the epoch for
+    # Elasticsearch to parse it correctly. Here we make that conversion.
+    date {
+        match => ["creation_time", "UNIX"]
+        target => "creation_time"
+    }
+
+    # OpenStack log levels are uppercase, and syslog are lowercase.
+    # Furthermore, syslog has more log levels that OpenStack. To avoid
+    # mapping syslog log levels to OpenStack log levels, we standardise
+    # on the syslog style here.
+    if [log][dimensions][log_level] {
+        mutate {
+            lowercase => [ "[log][dimensions][log_level]" ]
+        }
+    }
+}
+
 output {
     elasticsearch {
         index => "monasca-%{[meta][tenantId]}-%{+YYYY.MM.dd}"

ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2

@@ -1,46 +0,0 @@
-# Provide input/output streams for transforming Monasca logs.
-# Filters should be provided in other configuration files.
-
-input {
-    kafka {
-        bootstrap_servers => "{{ monasca_kafka_servers }}"
-        topics => ["{{ monasca_raw_logs_topic }}"]
-        group_id => "log_transformer"
-        consumer_threads => "{{ monasca_log_pipeline_threads }}"
-        codec => json
-    }
-}
-
-filter {
-    # Update the timestamp of the event based on the time in the message.
-    date {
-        match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss Z", "ISO8601"]
-        remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ]
-    }
-
-    # Monasca Log API adds a timestamp when it processes a log entry. This
-    # timestamp needs to be converted from seconds since the epoch for
-    # Elasticsearch to parse it correctly. Here we make that conversion.
-    date {
-        match => ["creation_time", "UNIX"]
-        target => "creation_time"
-    }
-
-    # OpenStack log levels are uppercase, and syslog are lowercase.
-    # Furthermore, syslog has more log levels that OpenStack. To avoid
-    # mapping syslog log levels to OpenStack log levels, we standardise
-    # on the syslog style here.
-    if [log][dimensions][log_level] {
-        mutate {
-            lowercase => [ "[log][dimensions][log_level]" ]
-        }
-    }
-}
-
-output {
-    kafka {
-        codec => json
-        bootstrap_servers => "{{ monasca_kafka_servers }}"
-        topic_id => "{{ monasca_transformed_logs_topic }}"
-    }
-}

ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2

@@ -1,25 +0,0 @@
-{
-    "command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-transformer -f /etc/logstash/conf.d/log-transformer.conf",
-    "config_files": [
-        {
-            "source": "{{ container_config_directory }}/log-transformer.conf",
-            "dest": "/etc/logstash/conf.d/log-transformer.conf",
-            "owner": "logstash",
-            "perm": "0600"
-        },
-        {
-            "source": "{{ container_config_directory }}/logstash_patterns/*",
-            "dest": "/etc/logstash/conf.d/patterns/",
-            "owner": "logstash",
-            "perm": "0600",
-            "optional": true
-        }
-    ],
-    "permissions": [
-        {
-            "path": "/var/log/kolla/logstash",
-            "owner": "logstash:kolla",
-            "recurse": true
-        }
-    ]
-}

doc/source/reference/logging-and-monitoring/monasca-guide.rst

@@ -318,6 +318,42 @@ Apply the password changes by running the following command:
 
    kolla-ansible reconfigure -t monasca
 
+Cleanup
+~~~~~~~
+
+From time-to-time it may be necessary to manually invoke the Monasca cleanup
+command. If this is required during an upgrade it will be mentioned in the
+release notes. It may also be necessary to run the cleanup command when
+disabling certain parts of the Monasca pipeline. A full list of scenarios in
+which you must run the cleanup command is given below:
+
+- Upgrading from Victoria to Wallaby to remove the unused Monasca Log
+  Transformer service
+
+The cleanup command can be invoked from the Kolla Ansible CLI, for example:
+
+.. code-block:: console
+
+   kolla-ansible monasca_cleanup
+
+Following cleanup, you may also choose to remove unused container volumes.
+It is recommended to run this manually on each Monasca service host. Note
+that `docker prune` will indiscriminately remove all unused volumes,
+which may not always be what you want. If you wish to keep a subset of
+unused volumes, you can remove them individually.
+
+To remove all unused volumes on a host:
+
+.. code-block:: console
+
+   docker prune
+
+To remove a single unused volume, run for example:
+
+.. code-block:: console
+
+   docker volume rm monasca_log_transformer_data
+
 System requirements and performance impact
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -335,9 +371,8 @@ Monasca will deploy the following Docker containers:
 * Monasca Agent Statsd
 * Monasca API
 * Monasca Log API
-* Monasca Log Transformer (Logstash)
 * Monasca Log Metrics (Logstash)
-* Monasca Log Perister (Logstash)
+* Monasca Log Persister (Logstash)
 * Monasca Notification
 * Monasca Persister
 * Monasca Thresh (Apache Storm topology)

releasenotes/notes/remove-monasca-log-transformer-bdcb3ef1cdbc1611.yaml

@@ -0,0 +1,24 @@
+---
+upgrade:
+  - |
+    Monasca Log Transformer has been merged with Monasca Log Persister to
+    improve performance and reduce resource consumption. Any custom Monasca
+    Log Transformer configuration should be either merged into Monasca Log
+    Persister configuration, or moved outside of the Monasca pipeline, for
+    example, to Fluentd. Any custom Monasca Log Metrics config will also
+    need to be updated to read from the raw logs pipeline, rather than the
+    transformed logs pipeline. The transformed logs pipeline will be
+    removed from Kafka automatically, as will any log transformer containers.
+    There will be a short interruption to logging services whilst the
+    pipeline is updated. During this time it's likely that a small window
+    of logs will be lost from the transformed logs Kafka queue. If this
+    is a problem, the Monasca API should be stopped on all nodes prior to
+    upgrading Monasca. This will allow the transformed logs topic to drain
+    into Elasticsearch before the pipeline is reconfigured. Services such
+    as Fluentd, which post logs to the Monasca API, should buffer logs
+    whilst this happens up to the maximum configured buffer. Note that
+    there may be other services forwarding logs, and these will need to be
+    inspected independently. The Log Transformer volumes will remain on
+    the monitoring nodes and can be manually removed as described in the
+    `documentation
+    <https://docs.openstack.org/kolla-ansible/latest/reference/logging-and-monitoring/monasca-guide.html#cleanup>`__.

tools/kolla-ansible

@@ -133,6 +133,7 @@ Commands:
     mariadb_backup       Take a backup of MariaDB databases
                              --full (default)
                              --incremental
+    monasca_cleanup      Remove unused containers for the Monasca service
     bootstrap-servers    Bootstrap servers with kolla deploy dependencies
     destroy              Destroy Kolla containers, volumes and host configuration
                              --include-images to also destroy Kolla images
@@ -175,6 +176,7 @@ prechecks
 check
 mariadb_recovery
 mariadb_backup
+monasca_cleanup
 bootstrap-servers
 destroy
 deploy
@@ -359,6 +361,11 @@ case "$1" in
         EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=backup -e mariadb_backup_type=${BACKUP_TYPE}"
         PLAYBOOK="${BASEDIR}/ansible/mariadb_backup.yml"
         ;;
+(monasca_cleanup)
+        ACTION="Cleanup unused Monasca services"
+        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=cleanup"
+        PLAYBOOK="${BASEDIR}/ansible/monasca_cleanup.yml"
+        ;;
 (destroy)
         ACTION="Destroy Kolla containers, volumes and host configuration"
         PLAYBOOK="${BASEDIR}/ansible/destroy.yml"