add filter log parser script, for https://github.com/opnsense/core/issues/1788

2026-03-15 00:54:41 +00:00 · 2017-10-18 18:54:43 +02:00 · 2017-10-18 18:54:43 +02:00 · 97fb9662bc
commit 97fb9662bc
parent 65a6fb2cda
1 changed files with 113 additions and 0 deletions
--- a/src/opnsense/scripts/filter/read_log.py
+++ b/src/opnsense/scripts/filter/read_log.py
@ -0,0 +1,113 @@
+#!/usr/local/bin/python2.7
+
+"""
+    Copyright (c) 2017 Ad Schellevis <ad@opnsense.org>
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+
+    --------------------------------------------------------------------------------------
+    read filter log, limit by number of records or last received digest (md5 hash of row)
+"""
+import os
+import sys
+import md5
+import argparse
+import ujson
+sys.path.insert(0, "/usr/local/opnsense/site-python")
+from log_helper import reverse_log_reader, fetch_clog
+from params import update_params
+
+filter_log = '/var/log/filter.log'
+
+# define log layouts, every endpoint contains all options
+# source : https://github.com/opnsense/ports/blob/master/opnsense/filterlog/files/description.txt
+fields_general = 'rulenr,subrulenr,anchorname,ridentifier,interface,reason,action,dir,version'.split(',')
+fields_ipv4 = fields_general + 'tos,ecn,ttl,id,offset,flags,proto,protoname,length,src,dst'.split(',')
+fields_ipv4_udp = fields_ipv4 + 'srcport,dstport,datalen'.split(',')
+fields_ipv4_tcp = fields_ipv4 + 'srcport,dstport,datalen,flags,error_options'.split(',')
+fields_ipv4_carp = fields_ipv4 + 'type,ttl,vhid,version,advskew,advbase'.split(',')
+
+fields_ipv6 = fields_general + 'class,flowlabel,hlim,next-header,next,payload-length,src,dst'.split(',')
+fields_ipv6_udp = fields_ipv6 + 'srcport,dstport,datalen'.split(',')
+fields_ipv6_tcp = fields_ipv6 + 'srcport,dstport,datalen,flags,error_options'.split(',')
+fields_ipv6_carp = fields_ipv6 + 'type,ttl,vhid,version2,advskew,advbase'.split(',')
+
+def update_rule(target, ruleparts, spec):
+    """ update target rule with parts in spec
+        :param target: target rule
+        :param ruleparts: list of rule items
+        :param spec: full rule specification, depending on protocol and version
+    """
+    while len(target) < len(spec) and len(ruleparts) > 0:
+        target[spec[len(target)]] = ruleparts.pop(0)
+    # full spec
+    target['__spec__'] = spec
+
+if __name__ == '__main__':
+    # read parameters
+    parameters = {'limit': '0', 'digest': ''}
+    update_params(parameters)
+    parameters['limit'] = int(parameters['limit'])
+
+    result = list()
+    for record in reverse_log_reader(fetch_clog(filter_log)):
+        if record['line'].find('filterlog') > -1:
+            rule = dict()
+            # rule metadata (unique hash, hostname, timestamp)
+            tmp = record['line'].split('filterlog:')[0].split()
+            rule['__digest__'] = md5.new(record['line']).hexdigest()
+            rule['__host__'] = tmp.pop()
+            rule['__timestamp__'] =  ' '.join(tmp)
+
+            rulep = record['line'].split('filterlog:')[1].strip().split(',')
+            update_rule(rule, rulep, fields_general)
+
+            if 'version' in rule:
+                if rule['version'] == '4':
+                    update_rule(rule, rulep, fields_ipv4)
+                    if 'proto' in rule:
+                        if rule['proto'] == '17': # UDP
+                            update_rule(rule, rulep, fields_ipv4_udp)
+                        elif  rule['proto'] == '6': # TCP
+                            update_rule(rule, rulep, fields_ipv4_tcp)
+                        elif  rule['proto'] == '112': # CARP
+                            update_rule(rule, rulep, fields_ipv4_carp)
+                elif rule['version'] == '6':
+                    update_rule(rule, rulep, fields_ipv6)
+                    if 'next' in rule:
+                        if rule['next'] == '17': # UDP
+                            update_rule(rule, rulep, fields_ipv6_udp)
+                        elif  rule['next'] == '6': # TCP
+                            update_rule(rule, rulep, fields_ipv6_tcp)
+                        elif  rule['next'] == '112': # CARP
+                            update_rule(rule, rulep, fields_ipv6_carp)
+
+            result.append(rule)
+            # handle exit criteria, row limit or last digest
+            if parameters['limit'] != 0 and len(result) > parameters['limit']:
+                break
+            elif parameters['digest'].strip() != '' and parameters['digest'] == rule['__digest__']:
+                break
+
+
+    print(ujson.dumps(result))