From e66dbbd6eb8a9c113ca62e0efec5f03f8c49a4c4 Mon Sep 17 00:00:00 2001 From: Ad Schellevis Date: Sat, 8 Jun 2024 19:14:38 +0200 Subject: [PATCH] Firewall: Diagnostics: Sessions - refactor pftop output, move search to controller layer and implement cache. This commit should improve responsiveness of the sessions screen, since we needed to parse the full data in the previous version as well before returning it, this shouldn't be much slower on initial load. Only risk is the size of the generated json output, by moving the label parsing we replicate less data and reduce total size. --- .../Diagnostics/Api/FirewallController.php | 86 +++++++++++-------- src/opnsense/scripts/filter/lib/states.py | 55 +++--------- src/opnsense/scripts/filter/pftop.py | 37 +------- .../conf/actions.d/actions_filter.conf | 2 +- 4 files changed, 64 insertions(+), 116 deletions(-) diff --git a/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/FirewallController.php b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/FirewallController.php index f0a9da79f..e796114df 100644 --- a/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/FirewallController.php +++ b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/FirewallController.php @@ -1,7 +1,7 @@ request->isPost()) { $this->sessionClose(); - $filter = new SanitizeFilter(); - $searchPhrase = ''; - $ruleId = ''; - $sortBy = ''; - $itemsPerPage = $this->request->getPost('rowCount', 'int', 9999); - $currentPage = $this->request->getPost('current', 'int', 1); + $pftop = json_decode((new Backend())->configdpRun('filter diag top') ?? '', true) ?? []; - if ($this->request->getPost('ruleid', 'string', '') != '') { - $ruleId = $filter->sanitize($this->request->getPost('ruleid'), 'query'); + $clauses = []; + $networks = []; + foreach (preg_split('/\s+/', (string)$this->request->getPost('searchPhrase', null, '')) as $item) { + if (empty($item)) { + continue; + } elseif (Util::isSubnet($item)) { + $networks[] = $item; + } elseif (Util::isIpAddress($item)) { + $networks[] = $item . "/". (strpos($item, ':') ? '128' : '32'); + } else { + $clauses[] = $item; + } } - if ($this->request->getPost('searchPhrase', 'string', '') != '') { - $searchPhrase = $filter->sanitize($this->request->getPost('searchPhrase'), 'query'); - } - if ( - $this->request->has('sort') && - is_array($this->request->getPost("sort")) && - !empty($this->request->getPost("sort")) - ) { - $tmp = array_keys($this->request->getPost("sort")); - $sortBy = $tmp[0] . " " . $this->request->getPost("sort")[$tmp[0]]; - } + $ruleid = $this->request->getPost('ruleid', 'string', ''); + $labels = $pftop['metadata']['labels']; + $filter_funct = function (&$row) use ($networks, $labels, $ruleid) { + /* update record */ + if (isset($labels[$row['rule']])) { + $row['label'] = $labels[$row['rule']]['rid']; + $row['descr'] = $labels[$row['rule']]['descr']; + } - $response = (new Backend())->configdpRun('filter diag top', [$searchPhrase, $itemsPerPage, - ($currentPage - 1) * $itemsPerPage, $ruleId, $sortBy]); - $response = json_decode($response, true); - if ($response != null) { - return [ - 'rows' => $response['details'], - 'rowCount' => count($response['details']), - 'total' => $response['total_entries'], - 'current' => (int)$currentPage - ]; - } + if (!empty($ruleid) && trim($row['label']) != $ruleid) { + return false; + } + /* filter using network clauses*/ + if (empty($networks)) { + return true; + } + foreach (['dst_addr', 'src_addr', 'gw_addr'] as $addr) { + foreach ($networks as $net) { + if (Util::isIPInCIDR($row[$addr] ?? '', $net)) { + return true; + } + } + } + return false; + }; + + return $this->searchRecordsetBase( + $pftop['details'], + null, + null, + $filter_funct, + SORT_NATURAL | SORT_FLAG_CASE, + $clauses + ); } - return [ - 'rows' => [], - 'rowCount' => 0, - 'total' => 0, - 'current' => 0 - ]; } + /** * delete / drop a specific state by state+creator id */ diff --git a/src/opnsense/scripts/filter/lib/states.py b/src/opnsense/scripts/filter/lib/states.py index 65e0d385b..41e3c8abf 100755 --- a/src/opnsense/scripts/filter/lib/states.py +++ b/src/opnsense/scripts/filter/lib/states.py @@ -1,5 +1,5 @@ """ - Copyright (c) 2015-2021 Ad Schellevis + Copyright (c) 2015-2024 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -247,18 +247,21 @@ def query_states(rule_label, filter_str): -def query_top(rule_label, filter_str): +def query_top(): addr_parser = AddressParser() - result = list() - rule_labels = fetch_rule_labels() + result = { + 'details': [], + 'metadata': { + 'labels': fetch_rule_labels() + } + } + sp = subprocess.run( - ['/usr/local/sbin/pftop', '-w', '1000', '-b','-v', 'long','9999999999999'], + ['/usr/local/sbin/pftop', '-w', '1000', '-b','-v', 'long','200000'], capture_output=True, text=True ) - filter_net_clauses, filter_clauses = split_filter_clauses(filter_str) - for rownum, line in enumerate(sp.stdout.strip().split('\n')): parts = line.strip().split() if rownum >= 2 and len(parts) > 5: @@ -270,7 +273,7 @@ def query_top(rule_label, filter_str): 'dst_addr': addr_parser.split_ip_port(parts[3])['addr'], 'dst_port': addr_parser.split_ip_port(parts[3])['port'], 'gw_addr': None, - 'gw_port': None, + 'gw_port': None } if parts[4].count(':') > 2 or parts[4].count('.') > 2: record['gw_addr'] = addr_parser.split_ip_port(parts[4])['addr'] @@ -301,12 +304,6 @@ def query_top(rule_label, filter_str): record['bytes'] = 0 record['avg'] = int(parts[idx+5]) if parts[idx+5].isdigit() else 0 record['rule'] = parts[idx+6] - if record['rule'] in rule_labels: - record['label'] = rule_labels[record['rule']]['rid'] - record['descr'] = rule_labels[record['rule']]['descr'] - else: - record['label'] = None - record['descr'] = None for timefield in ['age', 'expire']: if ':' in record[timefield]: tmp = record[timefield].split(':') @@ -318,35 +315,7 @@ def query_top(rule_label, filter_str): else: record[timefield] = 0 - if rule_label != "" and record['label'].lower().find(rule_label) == -1: - # label - continue - elif filter_clauses or filter_net_clauses: - match = False - for filter_net in filter_net_clauses: - try: - match = False - for field in ['src_addr', 'dst_addr', 'gw_addr']: - port_field = "%s_port" % field[0:3] - if record[field] is not None and addr_parser.overlaps(filter_net[0], record[field]): - if filter_net[1] is None or filter_net[1] == record[port_field]: - match = True - if not match: - break - except: - continue - if not match: - continue - if filter_clauses: - search_line = " ".join(str(item) for item in filter(None, record.values())) - for filter_clause in filter_clauses: - if search_line.find(filter_clause) == -1: - match = False - break - if not match: - continue - - result.append(record) + result['details'].append(record) return result diff --git a/src/opnsense/scripts/filter/pftop.py b/src/opnsense/scripts/filter/pftop.py index cbc0b9a4e..1eed92c81 100755 --- a/src/opnsense/scripts/filter/pftop.py +++ b/src/opnsense/scripts/filter/pftop.py @@ -1,7 +1,7 @@ #!/usr/local/bin/python3 """ - Copyright (c) 2021 Ad Schellevis + Copyright (c) 2021-2024 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -26,41 +26,8 @@ POSSIBILITY OF SUCH DAMAGE. """ import ujson -import argparse from lib.states import query_top if __name__ == '__main__': - # parse input arguments - parser = argparse.ArgumentParser() - parser.add_argument('--filter', help='filter results', default='') - parser.add_argument('--limit', help='limit number of results', default='') - parser.add_argument('--offset', help='offset results', default='') - parser.add_argument('--label', help='label / rule id', default='') - parser.add_argument('--sort_by', help='sort by (field asc|desc)', default='') - inputargs = parser.parse_args() - - result = { - 'details': query_top(filter_str=inputargs.filter, rule_label=inputargs.label) - } - # sort results - if inputargs.sort_by.strip() != '' and len(result['details']) > 0: - sort_key = inputargs.sort_by.split()[0] - sort_desc = inputargs.sort_by.split()[-1] == 'desc' - if sort_key in result['details'][0]: - if type(result['details'][0][sort_key]) is int: - sorter = lambda k: k[sort_key] if sort_key in k else 0 - else: - sorter = lambda k: str(k[sort_key]).lower() if sort_key in k else '' - result['details'] = sorted(result['details'], key=sorter, reverse=sort_desc) - - result['total_entries'] = len(result['details']) - # apply offset and limit - if inputargs.offset.isdigit(): - result['details'] = result['details'][int(inputargs.offset):] - if inputargs.limit.isdigit() and len(result['details']) >= int(inputargs.limit): - result['details'] = result['details'][:int(inputargs.limit)] - - result['total'] = len(result['details']) - - print(ujson.dumps(result)) + print(ujson.dumps(query_top())) diff --git a/src/opnsense/service/conf/actions.d/actions_filter.conf b/src/opnsense/service/conf/actions.d/actions_filter.conf index 2d7d55097..3f6022c3f 100644 --- a/src/opnsense/service/conf/actions.d/actions_filter.conf +++ b/src/opnsense/service/conf/actions.d/actions_filter.conf @@ -97,8 +97,8 @@ message:request pf rules [diag.top] command:/usr/local/opnsense/scripts/filter/pftop.py -parameters: --filter=%s --limit=%s --offset=%s --label=%s --sort_by=%s type:script_output +cache_ttl:30 message:request pftop statistics [diag.info]