grokparsefailure_sysloginput

[CFT6Server]

I noticed that I am getting the error/tag "grokparsefailure_sysloginput" in most of my inputs and I am trying to figure out why. Our filters are based on types or hosts, so unless there is something wrong with all the filters. When the filters were built, I don't recall seeing this tag....

Example Screenshots

tag example 1.JPG

tag example 2.JPG

tag example 3.JPG

filter config

Code: Select all

# 
# Logstash Configuration File
# Dynamically created by Nagios Log Server
#
# DO NOT EDIT THIS FILE. IT WILL BE OVERWRITTEN.
#
# Created Thu, 17 Dec 2015 17:00:32 -0800
#

#
# Global filters
#

filter {
    if [program] == 'apache_access' {
        grok {
            match => [ 'message', '%{COMBINEDAPACHELOG}']
        }
        date {
            match => [ 'timestamp', 'dd/MMM/yyyy:HH:mm:ss Z' ]
        }
        mutate {
            replace => [ 'type', 'apache_access' ]
             convert => [ 'bytes', 'integer' ]
             convert => [ 'response', 'integer' ]
        }
    }
     
    if [program] == 'apache_error' {
        grok {
            match => [ 'message', '\[(?<timestamp>%{DAY:day} %{MONTH:month} %{MONTHDAY} %{TIME} %{YEAR})\] \[%{WORD:class}\] \[%{WORD:originator} %{IP:clientip}\] %{GREEDYDATA:errmsg}']
        }
        date {
            match => [ 'timestamp', 'dd/MMM/yyyy:HH:mm:ss Z' ]
        }
        mutate {
            replace => [ 'type', 'apache_error' ]
        }
    }
    if [host] =~ /10\.242\.12\.193/  {
      drop { } 
    }
    
    if [host] =~ /10\.242\.12\.194/  {
      drop { } 
    }

    if [prog] == "NetApp" {
       grok {
          match => ["message", "%{SYSLOG5424PRI}%{CISCOTIMESTAMP} %{SYSLOG5424SD:EventType}: %{GREEDYDATA:MainMessage}"]
       }
       date {
          match => ["CISCOTIMESTAMP", "MMM dd HH:mm:ss"]
       }
       mutate {
          rename => {"message" => "source_message"}
       }
       grok {
           match => ["EventType", "\[%{DATA:node}:%{GREEDYDATA:service}:%{DATA:sev}]"]
       }
       mutate {
           replace => { "severity_label" => "%{sev}" }
       }
       if "_grokparsefailure_sysloginput" in [tags]{
            mutate {
               remove_tag => ["_grokparsefailure_sysloginput"]
    	    }
    	}
    	mutate {
    	   rename => {"MainMessage" => "message"}
    	   rename => {"syslog5424_pri" => "EventID"}
    	   remove_field => ["MainMessage","prog","sev"]
    	}
    }
    if [type] == "eventlog" {
       grok {
          match => ["Hostname", "%{WORD:Hostname}"]
          overwrite => ["Hostname"]
        }
    }
    if [type] == "dmz-eventlog" {
       grok {
          match => ["Hostname", "%{WORD:Hostname}"]
          overwrite => ["Hostname"]
          add_tag => ["Event Log"]
        }
    }
    if [type] == "syslog" {
        if "ERRPT" in [message] {
    	    grok {
            match => ["message","%{SYSLOG5424PRI}%{CISCOTIMESTAMP} Message forwarded from %{WORD:Hostname}: %{GREEDYDATA:msg}:"]
    		}
            mutate {        
                add_tag => ["AIX ERRPT"]
                remove_field => ["syslog5424_pri"]
    			}
    		grok {
    			match => ["msg", "(?<garbage>\-.*\-) LABEL: %{WORD:label} IDENTIFIER: %{WORD:indentifier} Date/Time: %{WORD:day} %{CISCOTIMESTAMP:datetime} %{DATA:garbage} Sequence Number: %{INT:sequence_number} (?<newmessage>.*)"]
    		}
    		mutate {
    			rename => ["message","Source_Message"]
    			rename => ["newmessage","message"]
    			remove_field => ["day","garbage","msg","_grokparsefailure"]
    		}
    	}
    }
    if [type] == "firewall-syslog"  {
        grok {
            match => [ 'message', '%{SYSLOG5424PRI}1 %{DATA:garbage1} %{DATA:Zone} - - - - \%(?<format>.*): %{GREEDYDATA:newmessage}']
    		}
        grok {
            match => [ 
    			'message', '%{CISCOFW106001_1}',
    			'message', '%{CISCOFW106001_2}',
    			'message', '%{CISCOFW106006_106007_1}',
    			'message', '%{CISCOFW106006_106007_2}',
    			'message', '%{CISCOFW106006_106007_106010}',
    			'message', '%{CISCOFW106015}',
    			'message', '%{CISCOFW106021}',
    			'message', '%{CISCOFW106023}',
    			'message', '%{CISCOFW106100}',
    			'message', '%{CISCOFW110002}',
    			'message', '%{CISCOFW302010}',
    			'message', '%{CISCOFW302013_302014_302015_302016_1}',
    			'message', '%{CISCOFW302013_302014_302015_302016_2}',
    			'message', '%{CISCOFW302020_302021_1}',
    			'message', '%{CISCOFW302020_302021_2}',			
    			'message', '%{CISCOFW305011}',
    			'message', '%{CISCOFW313001_313004_313008}',
    			'message', '%{CISCOFW313005}',
    			'message', '%{CISCOFW402117}',
    			'message', '%{CISCOFW402119}',
    			'message', '%{CISCOFW419001}',
    			'message', '%{CISCOFW419002}',
    			'message', '%{CISCOFW500004}',
    			'message', '%{CISCOFW602303_602304_1}',
    			'message', '%{CISCOFW602303_602304_2}',
    			'message', '%{CISCOFW710001_710002_710003_710005_710006}',
    			'message', '%{CISCOFW713172}',
    			'message', '%{CISCOFW733100}',
    			'message', '%{CISCOFW106014}'
    			]
    		}
    	geoip {
    		source => "src_IP"
    		add_tag => ['GeoIP']
    		}
    	mutate {
    		remove_field => ['garbage1','garbage2','newmessage']
    		remove_tag => ['_grokparsefailure']		
    		}
    	}
    if [type] == "IPS" {
    	grok {
    		match => [
    		"message","\[%{DATA:header}\] %{DATA:signature} \[Classification: %{DATA:classification}\] \[Priority: %{INT:priority}\] \{%{WORD:protocol}\} %{IP:src_IP}:%{INT:src_port} -> %{IP:dst_IP}:%{INT:dst_port}",
    		"message","\[%{DATA:header}\] %{DATA:signature} \[Classification: %{DATA:classification}\] \[Priority: %{INT:priority}\] \{%{WORD:protocol}\} %{IP:src_IP} -> %{IP:dst_IP}",
    		"message","%{WORD:throw}: %{DATA:snort_message} %{IP:src_IP} %{INT:src_port} --> %{IP:dst_IP} %{INT:dst_port}"
    		]
    		}
    	mutate {
    		remove_field => ["throw"]
    		}
    	geoip {
    		source => "src_IP"
    		add_tag => ['GeoIP']
    		}
    	}
    if [type] == 'f5-syslog'  {
        grok {
            match => [
    			'message', '%{SYSLOG5424PRI}%{CISCOTIMESTAMP:datetime} %{WORD:Hostname} %{WORD:severity_label} %{WORD}\[%{INT:throw1}\]: Rule %{DATA:rule} \<%{WORD:http_action}\>: \=\=\=\= %{WORD:secure_type} %{IP:src_IP}\%%{INT:throw2}:%{INT:src_port}: %{DATA:secure_protocol}:%{DATA:encryption}, UA: %{GREEDYDATA:client_info}',
    			'message', '%{SYSLOG5424PRI}%{CISCOTIMESTAMP:datetime} %{WORD:Hostname} %{WORD:severity_label} %{WORD}\[%{INT:throw1}\]: Rule %{DATA:rule} \<%{WORD:http_action}\>: Start. Client %{IP:src_IP}\%%{INT:throw2}:%{INT:src_port} \-\> %{DATA:url} \(request\)',
    			'message', '%{SYSLOG5424PRI}%{CISCOTIMESTAMP:datetime} %{WORD:Hostname} %{WORD:severity_label} %{WORD}\[%{INT:throw1}\]: Rule %{DATA:rule} \<%{WORD:http_action}\>: =================ACCEPTED: %{IP:src_IP}\%%{INT:throw2}:%{INT:src_port}'
    			]
    		}
    	geoip {
    		source => "src_IP"
    		add_tag => ['GeoIP']
    		}
    	mutate {
    		remove_field => ['throw1','throw2']
    		remove_tag => ['_grokparsefailure']	
    		}
    	}
    if [type] == 'AD DNS' {
        if [message] == '' {
            drop { }
            }
        grok {
            match => [ 'message', '%{DATE_US} %{TIME} (?:AM|PM) %{WORD:dns_thread_id} %{WORD:dns_context}%{SPACE}%{WORD:dns_packet_id} %{WORD:protocol} %{WORD:dns_direction} %{IP:src_IP}%{SPACE}%{WORD:dns_xid}%{SPACE}(?:Q|R Q)%{SPACE}\[%{WORD:dns_hex_flags}%{SPACE}%{WORD:dns_hex_flags}%{SPACE}%{GREEDYDATA:dns_response}\]%{SPACE}%{WORD:dns_recordtype}%{SPACE}%{GREEDYDATA:dns_query_name}']
        }
    }
    if ([host] == '10.242.13.1' or [host] == '10.242.13.2' or [host] =~ /10\.242\.(3|4|5|48)\.(25[0-4]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]?)/) {
        grok {
            match => [ 
                'message', '(?<esxi_thread_id>[0-9A-Z]{8,8}) %{DATA:esxi_loglevel} \'%{DATA:esxi_service}\' %{DATA}\] \[%{DATA}::%{DATA:esxi_sub_service}\] %{GREEDYDATA:esxi_message}',
    			'message', '(?<esxi_thread_id>[0-9A-Z]{8,8}) %{DATA:esxi_loglevel} \'%{DATA:esxi_service}\' %{DATA}\] \[%{DATA:esxi_sub_service}\] %{GREEDYDATA:esxi_message}',
    			'message', '(?<esxi_thread_id>[0-9A-Z]{8,8}) %{DATA:esxi_loglevel} \'%{DATA:esxi_service}\'\ (.*)] %{GREEDYDATA:esxi_message}',
    			'message', '(?<esxi_thread_id>[0-9A-Z]{8,8}) %{DATA:esxi_loglevel} \'%{DATA:esxi_service}\'] %{GREEDYDATA:esxi_message}',
    			'message', '%{GREEDYDATA:esxi_message}'
    			]
    			add_tag => ['VMware']
    		}
    	mutate {
    		replace => [ 'message', '%{esxi_message}']
    		remove_field => ['esxi_message']
    		}
    	}
}

#
# Local filters
#

[CFT6Server]

Here's the input... as I read that this could be an issue with the syslog input in logstash 1.5 onwards.

Code: Select all

# 
# Logstash Configuration File
# Dynamically created by Nagios Log Server
#
# DO NOT EDIT THIS FILE. IT WILL BE OVERWRITTEN.
#
# Created Thu, 17 Dec 2015 17:00:32 -0800
#

#
# Global inputs
#

input {
    syslog {
        type => 'linux-syslog'
        tags => 'Linux'
        port => 5544
    }
    tcp {
        type => 'eventlog'
        port => 3515
        tags => 'Event Log'
        codec => json {
            charset => 'CP1252'
        }
    }
    tcp {
        type => 'import_raw'
        tags => 'import_raw'
        port => 2056
    }
    tcp {
        type => 'import_json'
        tags => 'import_json'
        port => 2057
        codec => json
    }
    syslog {
        type => 'syslog'
        port => 514
    }
    syslog {
        type => 'f5-syslog'
        tags => 'F5'
        port => 5644
    }
    syslog {
        type => 'firewall-syslog'
        tags => 'Firewall'
        port => 5645
    }
    syslog {
        type => 'infoblox-syslog'
        tags => 'InfoBlox'
        port => 5646
    }
    tcp {
        type => 'dmz-eventlog'
        port => 3516
        tags => 'DMZ'
        codec => json {
            charset => 'CP1252'
        }
    }
    syslog {
        type => 'IPS'
        tags => 'IPS'
        port => 5647
    }
    tcp {
        type => 'AD DNS'
        port => 3517
        tags => 'AD DNS'
        codec => json {
            charset => 'CP1252'
        }
    }
}

#
# Local inputs
#

[jolson]

This is almost always due to the way that the 'syslog' input operates. This blog contains fantastic information on the subject: http://kartar.net/2014/09/when-logstash ... -go-wrong/

I recommend using a bare tcp or udp input and designing your filter to suit your log as exactly as possible, as opposed to relying on the syslog input. One of the issues with the syslog input is that I've seen it drop logs if the logs don't conform to the syslog format precisely.

[CFT6Server]

Thanks. I was considering that as well. I will look to change that since I am doing custom filter/grok anyways.

However, I did noticed that the logs weren't getting processed.... so this is before my reboot of the node that's receiving the message.

Code: Select all

<13>Jan  4 00:05:26 10.242.82.56 37.in-addr.arpa. 53340 IN PTR ..

Then after reboot...
04-Jan-2016 14:06:19.012 client 192.168.7.45#27056: UDP: query: static.uol.com.br IN A response: SERVFAIL +E

See the difference in the message. The message is now showing up in full rather that getting completely cutoff....

I also saw that some input did not even get recorded at all. Then after reboot, those types are coming back in again. Perhaps this node is overloaded and running out of resources which resulted in logs dropping?

[jolson]

See the difference in the message. The message is now showing up in full rather that getting completely cutoff....

Interesting. Being cut off could have had to do with the syslog input - I haven't seen a log come in halfway before, unless maybe UDP and some packet loss was involved.

Then after reboot, those types are coming back in again. Perhaps this node is overloaded and running out of resources which resulted in logs dropping?

Possibly - lets take a closer look. Please show us the output of the following:

Code: Select all

top | head -n5
free -m
curl -XGET 'localhost:9200/_nodes/jvm?pretty'
grep HEAP /etc/sysconfig/logstash
grep HEAP /etc/sysconfig/elasticsearch