So the past day or so I have been working on getting some good detail
from my NSX Edge’s (ESG and DLR) and I have been able to get them
working very well. The Kibana dashboard looks great too. :) So I wanted
to share this out with anyone else who may be looking at doing the same
thing or just needing a way to get useful information out of NSX in
regards to firewall activity. I am also working on some of the other
logging facilities within NSX and will share those once I get those
tidied up.
Below is the code you will need to get started in setting up your
logstash.conf file.
filter {
if "syslog" in [tags] {
grok {
match => [
"message", "<%{POSINT}>%{SYSLOGTIMESTAMP} %{SYSLOGHOST:syslog_host_id} %{GREEDYDATA}"
]
}
if [syslog_host_id] =~ /.*?(nsvpx).*?(everythingshouldbevirtual.local)?/ {
mutate {
add_tag => [ "Netscaler" ]
}
}
if [syslog_host_id] =~ /.*?(pfsense).*?(everythingshouldbevirtual.local)?/ {
mutate {
add_tag => [ "PFSense" ]
}
}
if "vShield-edge" in [syslog_host_id] {
mutate {
add_tag => [ "NSX", "pre-processed" ]
}
}
if "ESG-RT-" in [syslog_host_id] {
mutate {
add_tag => [ "NSX", "pre-processed" ]
}
}
}
}
# Setting up VMware ESX(i) log parsing
filter {
if "VMware" in [tags] {
if "Section for VMware ESX" in [message] {
drop { }
}
multiline {
pattern => "-->"
what => "previous"
}
grok {
break_on_match => true
match => [
"message", "<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{SYSLOGPROG:syslog_program}: (?<message_body>(?<message_system_info>(?:\[%{DATA:message_thread_id} %{DATA:syslog_level} \'%{DATA:message_service}\'\ ?%{DATA:message_opID}])) \[%{DATA:message_service_info}]\ (?<syslog_message>(%{GREEDYDATA})))",
"message", "<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{SYSLOGPROG:syslog_program}: (?<message_body>(?<message_system_info>(?:\[%{DATA:message_thread_id} %{DATA:syslog_level} \'%{DATA:message_service}\'\ ?%{DATA:message_opID}])) (?<syslog_message>(%{GREEDYDATA})))",
"message", "<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{SYSLOGPROG:syslog_program}: %{GREEDYDATA:syslog_message}",
"message", "<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{GREEDYDATA:syslog_message}"
]
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "YYYY-MM-ddHH:mm:ss,SSS", "YYYY-MM-dd HH:mm:ss,SSS", "ISO8601" ]
timezone => "UTC"
}
if "_grokparsefailure" not in [tags] {
mutate {
replace => [ "host", "%{syslog_hostname}" ]
}
}
if "Device naa" in [message] {
grok {
break_on_match => false
match => [
"message", "Device naa.%{WORD:device_naa} performance has %{WORD:device_status}%{GREEDYDATA} of %{INT:datastore_latency_from}%{GREEDYDATA} to %{INT:datastore_latency_to}",
"message", "Device naa.%{WORD:device_naa} performance has %{WORD:device_status}%{GREEDYDATA} from %{INT:datastore_latency_from}%{GREEDYDATA} to %{INT:datastore_latency_to}",
"message", "Device naa.%{WORD:device_naa} path %{NOTSPACE:device_vmhba} %{GREEDYDATA:device_message}"
]
}
}
if "connectivity issues" in [message] {
grok {
match => [
"message", "Hostd: %{GREEDYDATA} : %{DATA:device_access} to volume %{DATA:device_id} %{DATA:datastore} (following|due to)"
]
}
}
if "WARNING" in [message] {
grok {
match => [
"message", "WARNING: %{GREEDYDATA:vmware_warning_msg}"
]
}
}
mutate {
add_tag => [ "pre-processed" ]
}
}
}
filter {
if "VMware" in [tags] and [syslog_program] == "prnxd" {
multiline {
pattern => "\|"
what => "previous"
}
}
}
# NSX Distributed Firewall parsing
filter {
if "VMware" in [tags] and [syslog_program] == "dfwpktlogs" {
if "match" in [message] {
grok {
match => [
"message", "dfwpktlogs:.%{WORD:af_value}.%{WORD:rule_reason}.%{WORD:rule_action}.%{DATA:ruleset_name}/%{INT:rule_id}.%{WORD:rule_direction}.%{INT:length}.%{WORD:protocol}.%{IP:src_ip}/%{INT:src_port}.*%{IP:dst_ip}/%{INT:dst_port}.*%{WORD:flag}",
"message", "dfwpktlogs:.%{WORD:af_value}.%{WORD:rule_reason}.%{WORD:rule_action}.%{DATA:ruleset_name}/%{INT:rule_id}.%{WORD:rule_direction}.%{INT:length}.%{WORD:protocol}.%{IP:src_ip}/%{INT:src_port}.*%{IP:dst_ip}/%{INT:dst_port}",
"message", "dfwpktlogs:.%{WORD:af_value}.%{WORD:rule_reason}.%{WORD:rule_action}.%{DATA:ruleset_name}/%{INT:rule_id}.%{WORD:rule_direction}.%{INT:length}.*PROTO.*%{INT}.%{IP:src_ip}.*%{IP:dst_ip}",
"message", "dfwpktlogs:.%{WORD:af_value}.%{NOTSPACE:rule_action}.%{DATA:ruleset_name}/%{INT:rule_id}.%{WORD:rule_direction}.*PROTO.*%{INT}.%{IP:src_ip}.*%{IP:dst_ip}"
]
}
}
if "match" not in [message] {
grok {
match => [
"message", "dfwpktlogs:.%{WORD:af_value}.%{NOTSPACE:rule_action}.%{DATA:ruleset_name}/%{INT:rule_id}.%{WORD:rule_direction}.%{WORD:protocol}.%{IP:src_ip}/%{INT:src_port}.*%{IP:dst_ip}/%{INT:dst_port}.%{NOTSPACE:rule}.%{NOTSPACE}",
"message", "dfwpktlogs:.%{WORD:af_value}.%{NOTSPACE:rule_action}.%{DATA:ruleset_name}/%{INT:rule_id}.%{WORD:rule_direction}.*PROTO.*%{INT}.%{IP:src_ip}.*%{IP:dst_ip}.%{NOTSPACE:rule}.%{NOTSPACE}"
]
}
}
mutate {
add_tag => [ "NSX-DFW" ]
}
}
}
# NSX Initial Parsing
filter {
if "NSX" in [tags] {
grok {
match => [
"message", "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}"
]
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ]
timezone => "UTC"
}
if !("_grokparsefailure" in [tags]) {
mutate {
replace => [ "host", "%{syslog_hostname}" ]
}
}
}
}
# NSX Firewall Rule parsing
filter {
if "NSX" in [tags] and [syslog_program] == "firewall[]" {
grok {
break_on_match => false
match => [
"message", ": \[%{DATA:nsx_tenant}]\: (%{DATA:nsx_fw_action}_%{INT:nsx_fw_ruletag}IN=%{DATA:in_int}) OUT=%{DATA:out_int}?.*MAC=(%{COMMONMAC:dst_mac}):(%{COMMONMAC:src_mac}).*SRC=%{IP:src_ip}.*DST=%{IP:dst_ip}.*PROTO=(%{WORD:protocol}).?*SPT=(%{INT:src_port}?.*DPT=%{INT:dst_port}?.*)",
"message", ": \[%{DATA:nsx_tenant}]\: (%{DATA:nsx_fw_action}_%{INT:nsx_fw_ruletag}IN=%{DATA:in_int}).*SRC=%{IP:src_ip}.*DST=%{IP:dst_ip}.*PROTO=(%{WORD:protocol}).?*SPT=(%{INT:src_port}?.*DPT=%{INT:dst_port}?.*)",
"message", ": \[%{DATA:nsx_tenant}]\: (%{DATA:nsx_fw_action}_%{INT:nsx_fw_ruletag}IN=%{DATA:in_int}) OUT=%{DATA:out_int}?.*MAC=(%{COMMONMAC:dst_mac}):(%{COMMONMAC:src_mac}).*SRC=%{IP:src_ip}.*DST=%{IP:dst_ip}.*PROTO=(%{WORD:protocol})"
]
}
geoip {
source => "src_ip"
target => "geoip"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float" ]
}
mutate {
add_tag => [ "NSX-FW" ]
}
}
}
And here is a quick dashboard view for now.
Enjoy!
