Merge pull request #174 from whyscream/168-new-patterns

New patterns from #168

Author: whyscream

50-filter-postfix.conf

@@ -161,6 +161,20 @@ filter {
             tag_on_failure => [ "_grok_postfix_virtual_nomatch" ]
             add_tag        => [ "_grok_postfix_success" ]
         }
+    } else if [program] =~ /^postfix.*\/postmap$/ {
+        grok {
+            patterns_dir   => "/etc/logstash/patterns.d"
+            match          => [ "message", "^%{POSTFIX_POSTMAP}$" ]
+            tag_on_failure => [ "_grok_postfix_postmap_nomatch" ]
+            add_tag        => [ "_grok_postfix_success" ]
+        }
+    } else if [program] =~ /^postfix.*\/postfix-script$/ {
+        grok {
+            patterns_dir   => "/etc/logstash/patterns.d"
+            match          => [ "message", "^%{POSTFIX_SCRIPT}$" ]
+            tag_on_failure => [ "_grok_postfix_script_nomatch" ]
+            add_tag        => [ "_grok_postfix_success" ]
+        }
     } else if [program] =~ /^postfix.*/ {
         mutate {
             add_tag => [ "_grok_postfix_program_nomatch" ]

README.md

@@ -49,6 +49,10 @@ Contributing
 
 I only have access to my own log samples, and my setup does not support or use every feature in postfix. If you miss anything, please open a pull request on github. If you're not very well versed in regular expressions, it's also fine to only submit sample unsupported log lines.
 
+Other guidelines:
+- There is no goal to parse every possible Postfix log line. The goal is to extract useful data from the logs in a generic way.
+- The target for data extraction is logging from a local server. There have been requests to parse SMTP replies from remote (Postfix) servers that are logged by the SMTP client (`postfix/smtp` program name). There is no way to parse these replies in a generic way, they differ from implementation to implementation (f.i. Postfix vs Exim) and from server to server (every admin can customize the message format). Parsing stock replies from remote Postfix servers could be done, but would be confusing since the messages don't originate from the local server. Requests for parsing these are not honoured. If you like to do that, implement it yourself, or start a separate project, I'd be happy to add a link to it. :)
+
 License
 -------
 

postfix.grok

@@ -97,6 +97,7 @@ POSTFIX_SMTP_SSLCONNERR SSL_connect error to %{POSTFIX_RELAY_INFO}: %{POSTFIX_LO
 POSTFIX_SMTP_LOSTCONN %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} with %{POSTFIX_RELAY_INFO}( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})?
 POSTFIX_SMTP_TIMEOUT %{POSTFIX_QUEUEID:postfix_queueid}: conversation with %{POSTFIX_RELAY_INFO} timed out( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})?
 POSTFIX_SMTP_RELAYERR %{POSTFIX_QUEUEID:postfix_queueid}: host %{POSTFIX_RELAY_INFO} said: %{GREEDYDATA:postfix_smtp_response} \(in reply to %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} command\)
+POSTFIX_SMTP_SSLAUTHERR %{POSTFIX_QUEUEID:postfix_queueid}: SASL authentication failed; server %{POSTFIX_RELAY_INFO} said: %{GREEDYDATA:postfix_smtp_response}
 POSTFIX_SMTP_UTF8 host %{POSTFIX_RELAY_INFO} offers SMTPUTF8 support, but not 8BITMIME
 POSTFIX_SMTP_PIX %{POSTFIX_QUEUEID:postfix_queueid}: enabling PIX workarounds: %{DATA:postfix_pix_workaround} for %{POSTFIX_RELAY_INFO}
 
@@ -120,7 +121,7 @@ POSTFIX_PIPE %{POSTFIX_PIPE_ANY}
 POSTFIX_POSTSCREEN %{POSTFIX_PS_CONNECT}|%{POSTFIX_PS_ACCESS}|%{POSTFIX_PS_NOQUEUE}|%{POSTFIX_PS_TOOBUSY}|%{POSTFIX_PS_CACHE}|%{POSTFIX_PS_DNSBL}|%{POSTFIX_PS_VIOLATIONS}|%{POSTFIX_WARNING}
 POSTFIX_DNSBLOG %{POSTFIX_DNSBLOG_LISTING}|%{POSTFIX_WARNING}
 POSTFIX_ANVIL %{POSTFIX_ANVIL_CONN_RATE}|%{POSTFIX_ANVIL_CONN_CACHE}|%{POSTFIX_ANVIL_CONN_COUNT}
-POSTFIX_SMTP %{POSTFIX_SMTP_DELIVERY}|%{POSTFIX_SMTP_CONNERR}|%{POSTFIX_SMTP_SSLCONNERR}|%{POSTFIX_SMTP_LOSTCONN}|%{POSTFIX_SMTP_TIMEOUT}|%{POSTFIX_SMTP_RELAYERR}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTP_UTF8}|%{POSTFIX_TLSVERIFICATION}|%{POSTFIX_SMTP_PIX}
+POSTFIX_SMTP %{POSTFIX_SMTP_DELIVERY}|%{POSTFIX_SMTP_CONNERR}|%{POSTFIX_SMTP_SSLCONNERR}|%{POSTFIX_SMTP_SSLAUTHERR}|%{POSTFIX_SMTP_LOSTCONN}|%{POSTFIX_SMTP_TIMEOUT}|%{POSTFIX_SMTP_RELAYERR}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTP_UTF8}|%{POSTFIX_TLSVERIFICATION}|%{POSTFIX_SMTP_PIX}
 POSTFIX_DISCARD %{POSTFIX_DISCARD_ANY}|%{POSTFIX_WARNING}
 POSTFIX_LMTP %{POSTFIX_SMTP}
 POSTFIX_PICKUP %{POSTFIX_KEYVALUE}
@@ -136,3 +137,5 @@ POSTFIX_LOCAL %{POSTFIX_KEYVALUE}|%{POSTFIX_WARNING}
 POSTFIX_VIRTUAL %{POSTFIX_SMTP_DELIVERY}
 POSTFIX_ERROR %{POSTFIX_ERROR_ANY}
 POSTFIX_POSTSUPER %{POSTFIX_POSTSUPER_ACTION}|%{POSTFIX_POSTSUPER_SUMMARY}
+POSTFIX_POSTMAP %{POSTFIX_WARNING}
+POSTFIX_SCRIPT %{POSTFIX_WARNING}

test/command_counter_data_0003.yaml

@@ -1,4 +1,3 @@
-
 pattern: ^%{POSTFIX_COMMAND_COUNTER_DATA}
 data: helo=1 mail=2 rcpt=1 data=1 rset=1 quit=1 commands=7
 results:

test/command_counter_data_0004.yaml

@@ -1,4 +1,3 @@
-
 pattern: ^%{POSTFIX_COMMAND_COUNTER_DATA}
 data: helo=1 quit=1 unknown=0/2 commands=2/4
 results:

test/postmap_0001.yaml

@@ -0,0 +1,5 @@
+pattern: ^%{POSTFIX_POSTMAP}$
+data: "warning: /etc/postfix/conf.d/users.db: duplicate entry: \"xxx@yyy.com\""
+results:
+    postfix_message_level: warning
+    postfix_message: "/etc/postfix/conf.d/users.db: duplicate entry: \"xxx@yyy.com\""

test/script_0001.yaml

@@ -0,0 +1,5 @@
+pattern: ^%{POSTFIX_SCRIPT}$
+data: "warning: symlink leaves directory: /etc/postfix/./makedefs.out"
+results:
+    postfix_message_level: warning
+    postfix_message: "symlink leaves directory: /etc/postfix/./makedefs.out"

test/smtp_0032.yaml

@@ -0,0 +1,7 @@
+pattern: "^%{POSTFIX_SMTP}$"
+data: "D0F29603B4: SASL authentication failed; server xyz.example.com[1.2.3.4] said: 535 5.7.8 Error: authentication failed: authentication failure"
+results:
+    postfix_queueid: D0F29603B4
+    postfix_relay_hostname: xyz.example.com
+    postfix_relay_ip: 1.2.3.4
+    postfix_smtp_response: "535 5.7.8 Error: authentication failed: authentication failure"