From e4aa3e4f5e12a3605b1cf1fdc7954e8f30ea81e1 Mon Sep 17 00:00:00 2001 From: Ad Schellevis Date: Tue, 20 Jul 2021 21:53:33 +0200 Subject: [PATCH] Firewall / Diagnostics - time to kill legacy diag_system_pftop.php and replace it with "Sessions", which offers insights into the top sessions on the firewall related to rules and network segments. --- .../Diagnostics/Api/FirewallController.php | 44 +++++ .../Diagnostics/FirewallController.php | 7 + .../mvc/app/models/OPNsense/Core/ACL/ACL.xml | 5 +- .../app/models/OPNsense/Core/Menu/Menu.xml | 2 +- .../views/OPNsense/Diagnostics/fw_pftop.volt | 140 +++++++++++++++ src/opnsense/scripts/filter/lib/states.py | 72 ++++++++ src/opnsense/scripts/filter/pftop.py | 66 +++++++ .../conf/actions.d/actions_filter.conf | 4 +- src/www/diag_system_pftop.php | 170 ------------------ 9 files changed, 335 insertions(+), 175 deletions(-) create mode 100644 src/opnsense/mvc/app/views/OPNsense/Diagnostics/fw_pftop.volt create mode 100755 src/opnsense/scripts/filter/pftop.py delete mode 100644 src/www/diag_system_pftop.php diff --git a/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/FirewallController.php b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/FirewallController.php index 269488a7f..527edc5d3 100644 --- a/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/FirewallController.php +++ b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/FirewallController.php @@ -202,6 +202,50 @@ class FirewallController extends ApiControllerBase return []; } + /** + * query pftop + */ + public function queryPfTopAction() + { + if ($this->request->isPost()) { + $this->sessionClose(); + $filter = new Filter([ + 'query' => function ($value) { + return preg_replace("/[^0-9,a-z,A-Z, ,\/,*,\-,_,.,\#]/", "", $value); + } + ]); + $searchPhrase = ''; + $ruleId = ''; + $sortBy = ''; + $itemsPerPage = $this->request->getPost('rowCount', 'int', 9999); + $currentPage = $this->request->getPost('current', 'int', 1); + + if ($this->request->getPost('ruleid', 'string', '') != '') { + $ruleId = $filter->sanitize($this->request->getPost('ruleid'), 'query'); + } + + if ($this->request->getPost('searchPhrase', 'string', '') != '') { + $searchPhrase = $filter->sanitize($this->request->getPost('searchPhrase'), 'query'); + } + if ($this->request->has('sort') && is_array($this->request->getPost("sort"))) { + $tmp = array_keys($this->request->getPost("sort")); + $sortBy = $tmp[0] . " " . $this->request->getPost("sort")[$tmp[0]]; + } + + $response = (new Backend())->configdpRun('filter diag top', [$searchPhrase, $itemsPerPage, + ($currentPage - 1) * $itemsPerPage, $ruleId, $sortBy]); + $response = json_decode($response, true); + if ($response != null) { + return [ + 'rows' => $response['details'], + 'rowCount' => count($response['details']), + 'total' => $response['total_entries'], + 'current' => (int)$currentPage + ]; + } + } + return []; + } /** * delete / drop a specific state by state+creator id */ diff --git a/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/FirewallController.php b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/FirewallController.php index 44c0551a2..bf42e7b6c 100644 --- a/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/FirewallController.php +++ b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/FirewallController.php @@ -59,4 +59,11 @@ class FirewallController extends IndexController { $this->view->pick('OPNsense/Diagnostics/fw_states'); } + /** + * firewall pftop + */ + public function pfTopAction() + { + $this->view->pick('OPNsense/Diagnostics/fw_pftop'); + } } diff --git a/src/opnsense/mvc/app/models/OPNsense/Core/ACL/ACL.xml b/src/opnsense/mvc/app/models/OPNsense/Core/ACL/ACL.xml index 7143eaf02..7504d30ca 100644 --- a/src/opnsense/mvc/app/models/OPNsense/Core/ACL/ACL.xml +++ b/src/opnsense/mvc/app/models/OPNsense/Core/ACL/ACL.xml @@ -147,9 +147,10 @@ - Diagnostics: pfTop + Diagnostics: Firewall sessions - diag_system_pftop.php* + ui/diagnostics/firewall/pf_top* + api/diagnostics/firewall/pf_top* diff --git a/src/opnsense/mvc/app/models/OPNsense/Core/Menu/Menu.xml b/src/opnsense/mvc/app/models/OPNsense/Core/Menu/Menu.xml index a5cf2519c..f19f1cff4 100644 --- a/src/opnsense/mvc/app/models/OPNsense/Core/Menu/Menu.xml +++ b/src/opnsense/mvc/app/models/OPNsense/Core/Menu/Menu.xml @@ -188,8 +188,8 @@ + - diff --git a/src/opnsense/mvc/app/views/OPNsense/Diagnostics/fw_pftop.volt b/src/opnsense/mvc/app/views/OPNsense/Diagnostics/fw_pftop.volt new file mode 100644 index 000000000..ce1108b66 --- /dev/null +++ b/src/opnsense/mvc/app/views/OPNsense/Diagnostics/fw_pftop.volt @@ -0,0 +1,140 @@ +{# + # Copyright (c) 2021 Deciso B.V. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without modification, + # are permitted provided that the following conditions are met: + # + # 1. Redistributions of source code must retain the above copyright notice, + # this list of conditions and the following disclaimer. + # + # 2. Redistributions in binary form must reproduce the above copyright notice, + # this list of conditions and the following disclaimer in the documentation + # and/or other materials provided with the distribution. + # + # THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + # AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + # OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + # POSSIBILITY OF SUCH DAMAGE. + #} + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
{{ lang._('state id') }}{{ lang._('Dir') }}{{ lang._('Proto') }}{{ lang._('Source') }}{{ lang._('Gateway') }}{{ lang._('Destination') }}{{ lang._('State') }}{{ lang._('Age (sec)') }}{{ lang._('Expires (sec)') }}{{ lang._('Pkts') }}{{ lang._('Bytes') }}{{ lang._('Avg') }}{{ lang._('Rule') }}
+
diff --git a/src/opnsense/scripts/filter/lib/states.py b/src/opnsense/scripts/filter/lib/states.py index bb5cda4b8..6ef2dd9e6 100755 --- a/src/opnsense/scripts/filter/lib/states.py +++ b/src/opnsense/scripts/filter/lib/states.py @@ -157,3 +157,75 @@ def query_states(rule_label, filter_str): record['state'] = parts[-1] return result + + +def query_top(rule_label, filter_str): + result = list() + rule_labels = fetch_rule_labels() + sp = subprocess.run(['/usr/local/sbin/pftop', '-w', '1000', '-b','-v', 'long','9999999999999'], capture_output=True, text=True) + header = None + try: + filter_network = ipaddress.ip_network(filter_str.strip()) + except ValueError: + filter_network = None + + for rownum, line in enumerate(sp.stdout.strip().split('\n')): + parts = line.strip().split() + if rownum >= 2 and len(parts) > 5: + record = { + 'proto': parts[0], + 'dir': parts[1].lower(), + 'src_addr': parse_address(parts[2])['addr'], + 'src_port': parse_address(parts[2])['port'], + 'dst_addr': parse_address(parts[3])['addr'], + 'dst_port': parse_address(parts[3])['port'], + 'gw_addr': None, + 'gw_port': None, + } + if parts[4].count(':') > 2 or parts[4].count('.') > 2: + record['gw_addr'] = parse_address(parts[4])['addr'] + record['gw_port'] = parse_address(parts[4])['port'] + idx = 5 + else: + idx = 4 + + record['state'] = parts[idx] + record['age'] = parts[idx+1] + record['expire'] = parts[idx+2] + record['pkts'] = int(parts[idx+3]) + record['bytes'] = int(parts[idx+4]) + record['avg'] = int(parts[idx+5]) + record['rule'] = parts[idx+6] + if record['rule'] in rule_labels: + record['label'] = rule_labels[record['rule']]['rid'] + record['descr'] = rule_labels[record['rule']]['descr'] + else: + record['label'] = None + record['descr'] = None + for timefield in ['age', 'expire']: + tmp = record[timefield].split(':') + record[timefield] = int(tmp[0]) * 3600 + int(tmp[1]) * 60 + int(tmp[2]) + + search_line = " ".join(str(item) for item in filter(None, record.values())) + if rule_label != "" and record['label'].lower().find(rule_label) == -1: + # label + continue + elif filter_network is not None: + try: + match = False + for field in ['src_addr', 'dst_addr', 'gateway']: + addr = ipaddress.ip_network(record[field]) + if field is not None and ipaddress.ip_network(filter_network).overlaps(addr): + match = True + break + if not match: + continue + except: + continue + elif filter_str != "" and search_line.lower().find(filter_str.lower()) == -1: + # apply filter when provided + continue + + result.append(record) + + return result diff --git a/src/opnsense/scripts/filter/pftop.py b/src/opnsense/scripts/filter/pftop.py new file mode 100755 index 000000000..cbc0b9a4e --- /dev/null +++ b/src/opnsense/scripts/filter/pftop.py @@ -0,0 +1,66 @@ +#!/usr/local/bin/python3 + +""" + Copyright (c) 2021 Ad Schellevis + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +""" +import ujson +import argparse +from lib.states import query_top + + +if __name__ == '__main__': + # parse input arguments + parser = argparse.ArgumentParser() + parser.add_argument('--filter', help='filter results', default='') + parser.add_argument('--limit', help='limit number of results', default='') + parser.add_argument('--offset', help='offset results', default='') + parser.add_argument('--label', help='label / rule id', default='') + parser.add_argument('--sort_by', help='sort by (field asc|desc)', default='') + inputargs = parser.parse_args() + + result = { + 'details': query_top(filter_str=inputargs.filter, rule_label=inputargs.label) + } + # sort results + if inputargs.sort_by.strip() != '' and len(result['details']) > 0: + sort_key = inputargs.sort_by.split()[0] + sort_desc = inputargs.sort_by.split()[-1] == 'desc' + if sort_key in result['details'][0]: + if type(result['details'][0][sort_key]) is int: + sorter = lambda k: k[sort_key] if sort_key in k else 0 + else: + sorter = lambda k: str(k[sort_key]).lower() if sort_key in k else '' + result['details'] = sorted(result['details'], key=sorter, reverse=sort_desc) + + result['total_entries'] = len(result['details']) + # apply offset and limit + if inputargs.offset.isdigit(): + result['details'] = result['details'][int(inputargs.offset):] + if inputargs.limit.isdigit() and len(result['details']) >= int(inputargs.limit): + result['details'] = result['details'][:int(inputargs.limit)] + + result['total'] = len(result['details']) + + print(ujson.dumps(result)) diff --git a/src/opnsense/service/conf/actions.d/actions_filter.conf b/src/opnsense/service/conf/actions.d/actions_filter.conf index e0cb6792a..bbd8fefd6 100644 --- a/src/opnsense/service/conf/actions.d/actions_filter.conf +++ b/src/opnsense/service/conf/actions.d/actions_filter.conf @@ -96,8 +96,8 @@ type:script_output message:request pf rules [diag.top] -command:/usr/local/sbin/pftop -parameters: -w 200 -b -o %s -v %s %s +command:/usr/local/opnsense/scripts/filter/pftop.py +parameters: --filter=%s --limit=%s --offset=%s --label=%s --sort_by=%s type:script_output message:request pftop statistics diff --git a/src/www/diag_system_pftop.php b/src/www/diag_system_pftop.php deleted file mode 100644 index ad76f701c..000000000 --- a/src/www/diag_system_pftop.php +++ /dev/null @@ -1,170 +0,0 @@ - - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, - * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -require_once("guiconfig.inc"); - -$sorttypes = array('age', 'bytes', 'dest', 'dport', 'exp', 'none', 'peak', 'pkt', 'rate', 'size', 'sport', 'src'); -$viewtypes = array('default', 'label', 'long', 'rules', 'size', 'speed', 'state', 'time'); -$numstates = array('50', '100', '200', '500', '1000', '99999999999'); - -if ($_SERVER['REQUEST_METHOD'] === 'POST') { - // fetch only valid input data (items from above lists) - $viewtype = 'default'; - $numstate = '200'; - $sorttype ='bytes'; - if (isset($_POST['viewtype']) && in_array($_POST['viewtype'], $viewtypes)) { - $viewtype = $_POST['viewtype']; - } - if (isset($_POST['states']) && in_array($_POST['states'], $numstates)) { - $numstate = $_POST['states']; - } - if (isset($_POST['sorttype']) && in_array($_POST['sorttype'], $sorttypes)) { - $sorttype = $_POST['sorttype']; - } - - // fetch pftop data - echo configdp_run('filter diag top', array($sorttype, $viewtype, $numstate)); - exit; -} - -include("head.inc"); - -?> - - - - -
-
-
-
-
- - - - - - - - - - - - - - - -
- - -
- -
- 		-
- -
-
-
-
-
-
-
-
-
-
- -