From 4f7995fe1ac80d39757333d9a4af0a9ce8adf2db Mon Sep 17 00:00:00 2001
From: Andrew Bonney <andrew.bonney@bbc.co.uk>
Date: Wed, 6 May 2020 14:19:21 +0100
Subject: [PATCH] logstash: add cisco ios/nexus syslog parsing to pipeline

Adds parsing for Cisco NXOS/IOS syslog format, along with the
RFC5424 variant some devices can use. Messages which match these
patterns are fingerprinted based upon the message and host to
de-duplicate them when storing in elasticsearch.

Change-Id: I42fd441913d2095997c3493c37515362a5d732fc
---
 .../roles/elastic_logstash/templates/extras   |  2 +
 .../templates/logstash-pipelines.yml.j2       | 74 ++++++++++++++++++-
 2 files changed, 75 insertions(+), 1 deletion(-)

diff --git a/elk_metrics_7x/roles/elastic_logstash/templates/extras b/elk_metrics_7x/roles/elastic_logstash/templates/extras
index 8d9454de..022df79b 100644
--- a/elk_metrics_7x/roles/elastic_logstash/templates/extras
+++ b/elk_metrics_7x/roles/elastic_logstash/templates/extras
@@ -8,3 +8,5 @@ SWIFTPROXY_ACCESS %{DATA:clientip} %{DATA:serverip} %{SWIFTPROXY_DATE:timestamp}
 
 KEYSTONE_SUBSECOND_TIMESTAMP %{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}
 STANDARD_TIMESTAMP %{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}
+
+CISCOTIMESTAMP_EXTEND (%{YEAR} )?%{CISCOTIMESTAMP}( %{TZ})?
diff --git a/elk_metrics_7x/templates/logstash-pipelines.yml.j2 b/elk_metrics_7x/templates/logstash-pipelines.yml.j2
index 6080b50f..748d4ebf 100644
--- a/elk_metrics_7x/templates/logstash-pipelines.yml.j2
+++ b/elk_metrics_7x/templates/logstash-pipelines.yml.j2
@@ -467,6 +467,69 @@
               remove_field => [ "event_day", "event_month", "event_year", "event_time", "timestamp" ]
             }
           }
+        } else if [@metadata][source_type] == "syslog" {
+          if [message] == "" {
+            drop { }
+          }
+          mutate {
+            # Handle duplicate messages sent to logstash instances
+            copy => { "message" => "fingerprint_source" }
+          }
+          grok {
+            # Match syslog patterns used by Cisco switches for later handling
+            patterns_dir => [ "/opt/logstash/patterns" ]
+            match => [
+              # RFC5424
+              "message", "%{SYSLOG5424PRI} %{NUMBER:syslog.version} %{TIMESTAMP_ISO8601:logdate} %{HOSTNAME:syslog.host} +%%{CISCO_REASON:syslog.facility_label}-%{INT:syslog.severity}-%{CISCO_REASON:syslog.facility_sub_label}: %{GREEDYDATA:message}",
+              "message", "%{SYSLOG5424PRI} %{NUMBER:syslog.version} %{TIMESTAMP_ISO8601:logdate} %{HOSTNAME:syslog.host} last message repeated %{INT:syslog.repeats} times",
+              # Cisco Default (NXOS/IOSXE)
+              "message", "%{SYSLOG5424PRI}(%{NUMBER:syslog.sequence})?:( %{NUMBER}:)? %{CISCOTIMESTAMP_EXTEND:logdate}: +%%{CISCO_REASON:syslog.facility_label}-%{INT:syslog.severity}-%{CISCO_REASON:syslog.facility_sub_label}: %{GREEDYDATA:message}",
+              "message", "%{SYSLOG5424PRI}(%{NUMBER:syslog.sequence})?:( %{NUMBER}:)? %{CISCOTIMESTAMP_EXTEND:logdate}: last message repeated %{INT:syslog.repeats} times"
+            ]
+            overwrite => [ "message" ]
+            add_tag => [ "cisco" ]
+          }
+          if "cisco" in [tags] {
+            if [syslog.repeats] {
+              mutate {
+                replace => { "message" => "last message repeated %{syslog.repeats} times" }
+              }
+            }
+            date {
+              match => [
+                "logdate",
+                # RFC5424
+                "ISO8601",
+                # Cisco Nexus and IOS variants (space padded dates for joda time)
+                # There are too many options to list them all
+                "yyyy MMM  d HH:mm:ss.SSS ZZZ",
+                "yyyy MMM d HH:mm:ss.SSS ZZZ",
+                "yyyy MMM  d HH:mm:ss ZZZ",
+                "yyyy MMM d HH:mm:ss ZZZ",
+                "MMM  d HH:mm:ss.SSS",
+                "MMM d HH:mm:ss.SSS",
+                "MMM  d HH:mm:ss",
+                "MMM d HH:mm:ss"
+              ]
+              remove_field => [ "logdate" ]
+            }
+            mutate {
+              rename => ["syslog5424_pri", "syslog.priority"]
+              copy => {"syslog.severity" => "syslog.severity_label"}
+            }
+            mutate {
+              gsub => [
+                "syslog.severity_label", "0", "Emergency",
+                "syslog.severity_label", "1", "Alert",
+                "syslog.severity_label", "2", "Critical",
+                "syslog.severity_label", "3", "Error",
+                "syslog.severity_label", "4", "Warning",
+                "syslog.severity_label", "5", "Notification",
+                "syslog.severity_label", "6", "Informational",
+                "syslog.severity_label", "7", "Debug"
+              ]
+            }
+          }
         }
       }
       if [source.ip] {
@@ -480,7 +543,16 @@
           source => "ip"
         }
       }
-      if [message] {
+      if [fingerprint_source] {
+        fingerprint {
+          source => [ "host", "fingerprint_source" ]
+          target => "[@metadata][fingerprint]"
+          method => "SHA1"
+          key => "{{ logstash_elasticsearch_endpoints | to_uuid }}"
+          concatenate_sources => true
+          remove_field => [ "fingerprint_source" ]
+        }
+      } else if [message] {
         fingerprint {
           id => "setSHA1"
           target => "[@metadata][fingerprint]"