This is a raw json of one of the last events, after I applied and set the filter matching on info:
Code: Select all
{
"_index": "logstash-2015.04.01",
"_type": "syslog-f5",
"_id": "yCzvmzT4QCS2r35FqRt-LQ",
"_score": null,
"_source": {
"message": "<155>Apr 1 15:43:51 slot1/ca_f5_1_vir_pr err dcc[13273]: 01310033:3: [SECEV] Request blocked, violations: Attcak signature detected. HTTP protocol compliance sub violations: N/A. Evasion techniques sub violations: N/A. Web services security sub violations: N/A. Virus name: N/A. Support id: 2913722834563104156, source ip: 212.123.0.8, xff ip: 212.123.0.8, source port: 44601, destination ip: 20.20.1.138, destination port: 80, route_domain: 0, HTTP classifier: /Common/F5_External_1_LAMP_Policy, scheme HTTP, geographic location: <BE>, request: <GET / HTTP/1.1\\r\\nTE: deflate,gzip;q=0.3\\r\\nConnection: TE, close\\r\\nHost: www.digipolis.be\\r\\nUser-Agent: libwww-perl/6.04\\r\\n>, username: <N/A>, session_id: <1b5b12e54080e419>\n",
"@version": "1",
"@timestamp": "2015-04-01T13:43:51.838Z",
"type": "syslog-f5",
"host": "20.40.24.159",
"tags": [
"grokked"
],
"priority": 13,
"severity": 5,
"fcaility": 1,
"fcaility_label": "user-level",
"severity_label": "Notice",
"syslog5424_pri": "155",
"logsource": "ca_f5_1_vir_pr",
"loglevel": "err",
"program": "dcc",
"pid": "13273",
"info": "01310033:3: [SECEV] Request blocked, violations: Attcak signature detected. HTTP protocol compliance sub violations: N/A. Evasion techniques sub violations: N/A. Web services security sub violations: N/A. Virus name: N/A. Support id: 2913722834563104156, source ip: 212.123.0.8, xff ip: 212.123.0.8, source port: 44601, destination ip: 20.20.1.138, destination port: 80, route_domain: 0, HTTP classifier: /Common/F5_External_1_LAMP_Policy, scheme HTTP, geographic location: <BE>, request: <GET / HTTP/1.1\\r\\nTE: deflate,gzip;q=0.3\\r\\nConnection: TE, close\\r\\nHost: www.digipolis.be\\r\\nUser-Agent: libwww-perl/6.04\\r\\n>, username: <N/A>, session_id: <1b5b12e54080e419>"
},
"highlight": {
"message": [
"<155>Apr 1 15:43:51 slot1/ca_f5_1_vir_pr @start-highlight@err@end-highlight@ @start-highlight@dcc@end-highlight@[13273]: 01310033:3: [SECEV] Request blocked, violations: Attcak signature detected. HTTP protocol compliance sub violations: N/A. Evasion techniques sub violations: N/A. Web services security sub violations: N/A. Virus name: N/A. Support id: 2913722834563104156, source ip: 212.123.0.8, xff ip: 212.123.0.8, source port: 44601, destination ip: 20.20.1.138, destination port: 80, route_domain: 0, HTTP classifier: /Common/F5_External_1_LAMP_Policy, scheme HTTP, geographic location: <BE>, request: <GET / HTTP/1.1\\r\\nTE: deflate,gzip;q=0.3\\r\\nConnection: TE, close\\r\\nHost: www.digipolis.be\\r\\nUser-Agent: libwww-perl/6.04\\r\\n>, username: <N/A>, session_id: <1b5b12e54080e419>\n"
],
"loglevel": [
"@start-highlight@err@end-highlight@"
],
"loglevel.raw": [
"@start-highlight@err@end-highlight@"
],
"program.raw": [
"@start-highlight@dcc@end-highlight@"
],
"program": [
"@start-highlight@dcc@end-highlight@"
],
"type": [
"@start-highlight@syslog@end-highlight@-@start-highlight@f5@end-highlight@"
]
},
"sort": [
1427895831838,
1427895831838
]
}Code: Select all
#
# Logstash Configuration File
# Dynamically created by Nagios Log Server
#
# DO NOT EDIT THIS FILE. IT WILL BE OVERWRITTEN.
#
# Created Wed, 01 Apr 2015 15:49:30 +0200
#
#
# Global filters
#
filter {
if [program] == 'apache_access' {
grok {
match => [ 'message', '%{COMBINEDAPACHELOG}']
}
date {
match => [ 'timestamp', 'dd/MMM/yyyy:HH:mm:ss Z' ]
}
mutate {
replace => [ 'type', 'apache_access' ]
convert => [ 'bytes', 'integer' ]
convert => [ 'response', 'integer' ]
}
}
if [program] == 'apache_error' {
grok {
match => [ 'message', '\[(?<timestamp>%{DAY:day} %{MONTH:month} %{MONTHDAY} %{TIME} %{YEAR})\] \[%{WORD:class}\] \[%{WORD:originator} %{IP:clientip}\] %{GREEDYDATA:errmsg}']
}
mutate {
replace => [ 'type', 'apache_error' ]
}
}
if [type] == "syslog-brocade" {
grok {
match => [ "message", "<[\d]+>[a-z]+ [\d]+ [\d\:]+ %{IPV4:logsource}%{GREEDYDATA:program}: %{YEAR:year}\/%{MONTHNUM:month}\/%{MONTHDAY:day}-%{TIME:time}%{GREEDYDATA:data1}WWN %{IPV6:wwn}%{GREEDYDATA:data2}%{LOGLEVEL:loglevel}\, %{HOSTNAME:hostname}\, %{GREEDYDATA:info}" ]
remove_tag => "_grokparsefailure"
add_tag => "grokked"
}
}
if [type] == "syslog-f5" {
grok {
match => [ "message", "\A%{SYSLOG5424PRI}%{SYSLOGTIMESTAMP} slot1\/%{HOSTNAME:logsource} %{LOGLEVEL:loglevel} %{SYSLOGPROG}: %{GREEDYDATA:info}" ]
remove_tag => "_grokparsefailure"
add_tag => "grokked"
break_on_match => "false"
}
}
if [program] == "dcc" {
grok {
match => [ "info", "%{GREEDYDATA:info}, source ip: %{IP:sourceip}, xff ip: %{IP:xffip}, source port: %{NUMBER:sourceport}, destination ip: %{IP:destinationip}, destination port: %{NUMBER:destinationport}, route_domain: %{NUMBER:routedomain}, HTTP classifier: %{GREEDYDATA:httpclassifier}, geographic location: , request: %{GREEDYDATA:request}User-Agent: %{GREEDYDATA:useragent}, username: %{GREEDYDATA:username}, session_id: %{GREEDYDATA:sessionid}",
"info", "%{GREEDYDATA:info}, source ip: %{IP:sourceip}, xff ip: %{IP:xffip}, source port: %{NUMBER:sourceport}, destination ip: %{IP:destinationip}, destination port: %{NUMBER:destinationport}, route_domain: %{NUMBER:routedomain}, HTTP classifier: %{GREEDYDATA:httpclassifier}, geographic location: , request: %{GREEDYDATA:request}, username: %{GREEDYDATA:username}, session_id: %{GREEDYDATA:sessionid}" ]
}
#
# Local filters
#
So i'm still stuck. Some basic question I have in order to continue testing:
Code: Select all
break_on_match => "false" or break_on_match => falseif [program] == "dcc" {
grok {
match => [ "info", "%{GREEDYDATA:info}, source ip: %{IP:sourceip}, xff ip: %{IP:xffip}, source port: %{NUMBER:sourceport}, destination ip: %{IP:destinationip}, destination port: %{NUMBER:destinationport}, route_domain: %{NUMBER:routedomain}, HTTP classifier: %{GREEDYDATA:httpclassifier}, geographic location: , request: %{GREEDYDATA:request}User-Agent: %{GREEDYDATA:useragent}, username: %{GREEDYDATA:username}, session_id: %{GREEDYDATA:sessionid}",
"info", "%{GREEDYDATA:info}, source ip: %{IP:sourceip}, xff ip: %{IP:xffip}, source port: %{NUMBER:sourceport}, destination ip: %{IP:destinationip}, destination port: %{NUMBER:destinationport}, route_domain: %{NUMBER:routedomain}, HTTP classifier: %{GREEDYDATA:httpclassifier}, geographic location: , request: %{GREEDYDATA:request}, username: %{GREEDYDATA:username}, session_id: %{GREEDYDATA:sessionid}" ]
}
When I use http://grokconstructor.appspot.com/do/match#result to test the filter, i have a match.. (with of course the extra grok pattern
Code: Select all
HOSTNAME2 \b(?:[_0-9A-Za-z][_0-9A-Za-z-]{0,62})(?:\.(?:[_0-9A-Za-z][_0-9A-Za-z-]{0,62}))*(\.?|\b)Tx and grtz