Hi again,
unfortunately our filters still ain't working as expected.
they don't seem to be triggered reliable.
events of the same type from another host get filtered correctly. (and i expect that these examples eventually get filtered correctly after a while without further action)
Anybody has a hint what might be wrong in our setup?
thanks in advance
Mike
here is an example event:
Code: Select all
{
"_index": "logstash-2015.02.24",
"_type": "eventlog",
"_id": "GQkKhvz0Qhu3I6swxsQ2qA",
"_score": null,
"_source": {
"EventReceivedTime": "2015-02-24 10:15:16",
"SourceModuleName": "apache_access_custom_asp",
"SourceModuleType": "im_file",
"message": "1.2.3.4|192.168.131.186|-|[24/Feb/2015:10:15:16 +0100]|GET|/somerequest|200|32178|\"-\"|\"Java/1.7.0_55\"|my.vhost.local",
"@version": "1",
"@timestamp": "2015-02-24T09:15:16.855Z",
"host": "192.168.131.186",
"type": "eventlog"
},
"sort": [
1424769316855,
1424769316855
]
}
here is our complete filter:
Code: Select all
#
# Logstash Configuration File
# Dynamically created by Nagios Log Server
#
# DO NOT EDIT THIS FILE. IT WILL BE OVERWRITTEN.
#
# Created Tue, 24 Feb 2015 10:19:09 +0100
#
#
# Global filters
#
filter {
if [program] == 'apache_access' {
grok {
match => [ 'message', '%{COMBINEDAPACHELOG}']
}
date {
match => [ 'timestamp', 'dd/MMM/yyyy:HH:mm:ss Z' ]
}
mutate {
replace => [ 'type', 'apache_access' ]
convert => [ 'bytes', 'integer' ]
convert => [ 'response', 'integer' ]
}
}
if 'apache_access_common' in [SourceModuleName] {
grok {
match => [ 'message', '%{COMMONAPACHELOG}']
}
date {
match => [ 'timestamp', 'dd/MMM/yyyy:HH:mm:ss Z' ]
}
mutate {
replace => [ 'type', 'apache_access' ]
convert => [ 'bytes', 'integer' ]
convert => [ 'response', 'integer' ]
}
}
if [SourceModuleName] == 'apache_access_custom_app' {
grok {
break_on_match => false
match => [ 'message', '%{IPORHOST:clientip}\|%{IPORHOST:hostip}\|%{USER:auth}\|\[%{HTTPDATE:timestamp}\]\|%{WORD:verb}\|%{GREEDYDATA:request}\|%{NUMBER:response}\|%{USER:bytes}\|%{QS:referrer}\|%{QS:agent}\|%{IPORHOST:vhost}']
match => [ 'request', '/%{WORD:directory}/%{GREEDYDATA}']
}
date {
match => [ 'timestamp', 'dd/MMM/yyyy:HH:mm:ss Z' ]
}
mutate {
replace => [ 'type', 'apache_access' ]
convert => [ 'bytes', 'integer' ]
convert => [ 'response', 'integer' ]
}
geoip {
source => 'clientip'
}
}
if [SourceModuleName] == 'apache_access_custom_asp' {
grok {
break_on_match => false
match => [ 'message', '%{IPORHOST:clientip}\|%{IPORHOST:hostip}\|%{USER:auth}\|\[%{HTTPDATE:timestamp}\]\|%{WORD:verb}\|%{GREEDYDATA:request}\|%{NUMBER:response}\|%{USER:bytes}\|%{QS:referrer}\|%{QS:agent}\|%{IPORHOST:vhost}']
match => [ 'request', '/%{WORD:directory}/%{GREEDYDATA}']
}
date {
match => [ 'timestamp', 'dd/MMM/yyyy:HH:mm:ss Z' ]
}
mutate {
replace => [ 'type', 'apache_access' ]
convert => [ 'bytes', 'integer' ]
convert => [ 'response', 'integer' ]
}
geoip {
source => 'clientip'
}
}
if 'apache_error' in [SourceModuleName] {
grok {
match => [ 'message', '\[(?<timestamp>%{DAY:day} %{MONTH:month} %{MONTHDAY} %{TIME} %{YEAR})\] %{GREEDYDATA:errmsg}']
}
date {
match => [ 'timestamp', 'EEE MMM dd HH:mm:ss.SSSSSS yyyy' ]
}
mutate {
replace => [ 'type', 'apache_error' ]
}
}
if [SourceModuleType] == 'im_mseventlog' {
mutate {
add_field => [ 'Channel', '%{FileName}' ]
}
}
}
#
# Local filters
#