grokparsefailure_sysloginput
Posted: Mon Jan 04, 2016 4:45 pm
I noticed that I am getting the error/tag "grokparsefailure_sysloginput" in most of my inputs and I am trying to figure out why. Our filters are based on types or hosts, so unless there is something wrong with all the filters. When the filters were built, I don't recall seeing this tag....
Example Screenshots filter config
Example Screenshots filter config
Code: Select all
#
# Logstash Configuration File
# Dynamically created by Nagios Log Server
#
# DO NOT EDIT THIS FILE. IT WILL BE OVERWRITTEN.
#
# Created Thu, 17 Dec 2015 17:00:32 -0800
#
#
# Global filters
#
filter {
if [program] == 'apache_access' {
grok {
match => [ 'message', '%{COMBINEDAPACHELOG}']
}
date {
match => [ 'timestamp', 'dd/MMM/yyyy:HH:mm:ss Z' ]
}
mutate {
replace => [ 'type', 'apache_access' ]
convert => [ 'bytes', 'integer' ]
convert => [ 'response', 'integer' ]
}
}
if [program] == 'apache_error' {
grok {
match => [ 'message', '\[(?<timestamp>%{DAY:day} %{MONTH:month} %{MONTHDAY} %{TIME} %{YEAR})\] \[%{WORD:class}\] \[%{WORD:originator} %{IP:clientip}\] %{GREEDYDATA:errmsg}']
}
date {
match => [ 'timestamp', 'dd/MMM/yyyy:HH:mm:ss Z' ]
}
mutate {
replace => [ 'type', 'apache_error' ]
}
}
if [host] =~ /10\.242\.12\.193/ {
drop { }
}
if [host] =~ /10\.242\.12\.194/ {
drop { }
}
if [prog] == "NetApp" {
grok {
match => ["message", "%{SYSLOG5424PRI}%{CISCOTIMESTAMP} %{SYSLOG5424SD:EventType}: %{GREEDYDATA:MainMessage}"]
}
date {
match => ["CISCOTIMESTAMP", "MMM dd HH:mm:ss"]
}
mutate {
rename => {"message" => "source_message"}
}
grok {
match => ["EventType", "\[%{DATA:node}:%{GREEDYDATA:service}:%{DATA:sev}]"]
}
mutate {
replace => { "severity_label" => "%{sev}" }
}
if "_grokparsefailure_sysloginput" in [tags]{
mutate {
remove_tag => ["_grokparsefailure_sysloginput"]
}
}
mutate {
rename => {"MainMessage" => "message"}
rename => {"syslog5424_pri" => "EventID"}
remove_field => ["MainMessage","prog","sev"]
}
}
if [type] == "eventlog" {
grok {
match => ["Hostname", "%{WORD:Hostname}"]
overwrite => ["Hostname"]
}
}
if [type] == "dmz-eventlog" {
grok {
match => ["Hostname", "%{WORD:Hostname}"]
overwrite => ["Hostname"]
add_tag => ["Event Log"]
}
}
if [type] == "syslog" {
if "ERRPT" in [message] {
grok {
match => ["message","%{SYSLOG5424PRI}%{CISCOTIMESTAMP} Message forwarded from %{WORD:Hostname}: %{GREEDYDATA:msg}:"]
}
mutate {
add_tag => ["AIX ERRPT"]
remove_field => ["syslog5424_pri"]
}
grok {
match => ["msg", "(?<garbage>\-.*\-) LABEL: %{WORD:label} IDENTIFIER: %{WORD:indentifier} Date/Time: %{WORD:day} %{CISCOTIMESTAMP:datetime} %{DATA:garbage} Sequence Number: %{INT:sequence_number} (?<newmessage>.*)"]
}
mutate {
rename => ["message","Source_Message"]
rename => ["newmessage","message"]
remove_field => ["day","garbage","msg","_grokparsefailure"]
}
}
}
if [type] == "firewall-syslog" {
grok {
match => [ 'message', '%{SYSLOG5424PRI}1 %{DATA:garbage1} %{DATA:Zone} - - - - \%(?<format>.*): %{GREEDYDATA:newmessage}']
}
grok {
match => [
'message', '%{CISCOFW106001_1}',
'message', '%{CISCOFW106001_2}',
'message', '%{CISCOFW106006_106007_1}',
'message', '%{CISCOFW106006_106007_2}',
'message', '%{CISCOFW106006_106007_106010}',
'message', '%{CISCOFW106015}',
'message', '%{CISCOFW106021}',
'message', '%{CISCOFW106023}',
'message', '%{CISCOFW106100}',
'message', '%{CISCOFW110002}',
'message', '%{CISCOFW302010}',
'message', '%{CISCOFW302013_302014_302015_302016_1}',
'message', '%{CISCOFW302013_302014_302015_302016_2}',
'message', '%{CISCOFW302020_302021_1}',
'message', '%{CISCOFW302020_302021_2}',
'message', '%{CISCOFW305011}',
'message', '%{CISCOFW313001_313004_313008}',
'message', '%{CISCOFW313005}',
'message', '%{CISCOFW402117}',
'message', '%{CISCOFW402119}',
'message', '%{CISCOFW419001}',
'message', '%{CISCOFW419002}',
'message', '%{CISCOFW500004}',
'message', '%{CISCOFW602303_602304_1}',
'message', '%{CISCOFW602303_602304_2}',
'message', '%{CISCOFW710001_710002_710003_710005_710006}',
'message', '%{CISCOFW713172}',
'message', '%{CISCOFW733100}',
'message', '%{CISCOFW106014}'
]
}
geoip {
source => "src_IP"
add_tag => ['GeoIP']
}
mutate {
remove_field => ['garbage1','garbage2','newmessage']
remove_tag => ['_grokparsefailure']
}
}
if [type] == "IPS" {
grok {
match => [
"message","\[%{DATA:header}\] %{DATA:signature} \[Classification: %{DATA:classification}\] \[Priority: %{INT:priority}\] \{%{WORD:protocol}\} %{IP:src_IP}:%{INT:src_port} -> %{IP:dst_IP}:%{INT:dst_port}",
"message","\[%{DATA:header}\] %{DATA:signature} \[Classification: %{DATA:classification}\] \[Priority: %{INT:priority}\] \{%{WORD:protocol}\} %{IP:src_IP} -> %{IP:dst_IP}",
"message","%{WORD:throw}: %{DATA:snort_message} %{IP:src_IP} %{INT:src_port} --> %{IP:dst_IP} %{INT:dst_port}"
]
}
mutate {
remove_field => ["throw"]
}
geoip {
source => "src_IP"
add_tag => ['GeoIP']
}
}
if [type] == 'f5-syslog' {
grok {
match => [
'message', '%{SYSLOG5424PRI}%{CISCOTIMESTAMP:datetime} %{WORD:Hostname} %{WORD:severity_label} %{WORD}\[%{INT:throw1}\]: Rule %{DATA:rule} \<%{WORD:http_action}\>: \=\=\=\= %{WORD:secure_type} %{IP:src_IP}\%%{INT:throw2}:%{INT:src_port}: %{DATA:secure_protocol}:%{DATA:encryption}, UA: %{GREEDYDATA:client_info}',
'message', '%{SYSLOG5424PRI}%{CISCOTIMESTAMP:datetime} %{WORD:Hostname} %{WORD:severity_label} %{WORD}\[%{INT:throw1}\]: Rule %{DATA:rule} \<%{WORD:http_action}\>: Start. Client %{IP:src_IP}\%%{INT:throw2}:%{INT:src_port} \-\> %{DATA:url} \(request\)',
'message', '%{SYSLOG5424PRI}%{CISCOTIMESTAMP:datetime} %{WORD:Hostname} %{WORD:severity_label} %{WORD}\[%{INT:throw1}\]: Rule %{DATA:rule} \<%{WORD:http_action}\>: =================ACCEPTED: %{IP:src_IP}\%%{INT:throw2}:%{INT:src_port}'
]
}
geoip {
source => "src_IP"
add_tag => ['GeoIP']
}
mutate {
remove_field => ['throw1','throw2']
remove_tag => ['_grokparsefailure']
}
}
if [type] == 'AD DNS' {
if [message] == '' {
drop { }
}
grok {
match => [ 'message', '%{DATE_US} %{TIME} (?:AM|PM) %{WORD:dns_thread_id} %{WORD:dns_context}%{SPACE}%{WORD:dns_packet_id} %{WORD:protocol} %{WORD:dns_direction} %{IP:src_IP}%{SPACE}%{WORD:dns_xid}%{SPACE}(?:Q|R Q)%{SPACE}\[%{WORD:dns_hex_flags}%{SPACE}%{WORD:dns_hex_flags}%{SPACE}%{GREEDYDATA:dns_response}\]%{SPACE}%{WORD:dns_recordtype}%{SPACE}%{GREEDYDATA:dns_query_name}']
}
}
if ([host] == '10.242.13.1' or [host] == '10.242.13.2' or [host] =~ /10\.242\.(3|4|5|48)\.(25[0-4]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]?)/) {
grok {
match => [
'message', '(?<esxi_thread_id>[0-9A-Z]{8,8}) %{DATA:esxi_loglevel} \'%{DATA:esxi_service}\' %{DATA}\] \[%{DATA}::%{DATA:esxi_sub_service}\] %{GREEDYDATA:esxi_message}',
'message', '(?<esxi_thread_id>[0-9A-Z]{8,8}) %{DATA:esxi_loglevel} \'%{DATA:esxi_service}\' %{DATA}\] \[%{DATA:esxi_sub_service}\] %{GREEDYDATA:esxi_message}',
'message', '(?<esxi_thread_id>[0-9A-Z]{8,8}) %{DATA:esxi_loglevel} \'%{DATA:esxi_service}\'\ (.*)] %{GREEDYDATA:esxi_message}',
'message', '(?<esxi_thread_id>[0-9A-Z]{8,8}) %{DATA:esxi_loglevel} \'%{DATA:esxi_service}\'] %{GREEDYDATA:esxi_message}',
'message', '%{GREEDYDATA:esxi_message}'
]
add_tag => ['VMware']
}
mutate {
replace => [ 'message', '%{esxi_message}']
remove_field => ['esxi_message']
}
}
}
#
# Local filters
#