mirror of
https://github.com/lucaspalomodevelop/core.git
synced 2026-03-15 00:54:41 +00:00
Firewall: Diagnostics: Sessions - refactor pftop output, move search to controller layer and implement cache.
This commit should improve responsiveness of the sessions screen, since we needed to parse the full data in the previous version as well before returning it, this shouldn't be much slower on initial load. Only risk is the size of the generated json output, by moving the label parsing we replicate less data and reduce total size.
This commit is contained in:
Ad Schellevis
parent
d496eea29c
commit
e66dbbd6eb
@ -1,7 +1,7 @@
|
||||
<?php
|
||||
|
||||
/*
|
||||
* Copyright (C) 2017-2021 Deciso B.V.
|
||||
* Copyright (C) 2017-2024 Deciso B.V.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
@ -32,6 +32,7 @@ use OPNsense\Base\ApiControllerBase;
|
||||
use OPNsense\Core\Backend;
|
||||
use OPNsense\Core\Config;
|
||||
use OPNsense\Core\SanitizeFilter;
|
||||
use OPNsense\Firewall\Util;
|
||||
|
||||
/**
|
||||
* Class FirewallController
|
||||
@ -226,48 +227,59 @@ class FirewallController extends ApiControllerBase
|
||||
{
|
||||
if ($this->request->isPost()) {
|
||||
$this->sessionClose();
|
||||
$filter = new SanitizeFilter();
|
||||
$searchPhrase = '';
|
||||
$ruleId = '';
|
||||
$sortBy = '';
|
||||
$itemsPerPage = $this->request->getPost('rowCount', 'int', 9999);
|
||||
$currentPage = $this->request->getPost('current', 'int', 1);
|
||||
$pftop = json_decode((new Backend())->configdpRun('filter diag top') ?? '', true) ?? [];
|
||||
|
||||
if ($this->request->getPost('ruleid', 'string', '') != '') {
|
||||
$ruleId = $filter->sanitize($this->request->getPost('ruleid'), 'query');
|
||||
$clauses = [];
|
||||
$networks = [];
|
||||
foreach (preg_split('/\s+/', (string)$this->request->getPost('searchPhrase', null, '')) as $item) {
|
||||
if (empty($item)) {
|
||||
continue;
|
||||
} elseif (Util::isSubnet($item)) {
|
||||
$networks[] = $item;
|
||||
} elseif (Util::isIpAddress($item)) {
|
||||
$networks[] = $item . "/". (strpos($item, ':') ? '128' : '32');
|
||||
} else {
|
||||
$clauses[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
if ($this->request->getPost('searchPhrase', 'string', '') != '') {
|
||||
$searchPhrase = $filter->sanitize($this->request->getPost('searchPhrase'), 'query');
|
||||
}
|
||||
if (
|
||||
$this->request->has('sort') &&
|
||||
is_array($this->request->getPost("sort")) &&
|
||||
!empty($this->request->getPost("sort"))
|
||||
) {
|
||||
$tmp = array_keys($this->request->getPost("sort"));
|
||||
$sortBy = $tmp[0] . " " . $this->request->getPost("sort")[$tmp[0]];
|
||||
}
|
||||
$ruleid = $this->request->getPost('ruleid', 'string', '');
|
||||
$labels = $pftop['metadata']['labels'];
|
||||
$filter_funct = function (&$row) use ($networks, $labels, $ruleid) {
|
||||
/* update record */
|
||||
if (isset($labels[$row['rule']])) {
|
||||
$row['label'] = $labels[$row['rule']]['rid'];
|
||||
$row['descr'] = $labels[$row['rule']]['descr'];
|
||||
}
|
||||
|
||||
$response = (new Backend())->configdpRun('filter diag top', [$searchPhrase, $itemsPerPage,
|
||||
($currentPage - 1) * $itemsPerPage, $ruleId, $sortBy]);
|
||||
$response = json_decode($response, true);
|
||||
if ($response != null) {
|
||||
return [
|
||||
'rows' => $response['details'],
|
||||
'rowCount' => count($response['details']),
|
||||
'total' => $response['total_entries'],
|
||||
'current' => (int)$currentPage
|
||||
];
|
||||
}
|
||||
if (!empty($ruleid) && trim($row['label']) != $ruleid) {
|
||||
return false;
|
||||
}
|
||||
/* filter using network clauses*/
|
||||
if (empty($networks)) {
|
||||
return true;
|
||||
}
|
||||
foreach (['dst_addr', 'src_addr', 'gw_addr'] as $addr) {
|
||||
foreach ($networks as $net) {
|
||||
if (Util::isIPInCIDR($row[$addr] ?? '', $net)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
return $this->searchRecordsetBase(
|
||||
$pftop['details'],
|
||||
null,
|
||||
null,
|
||||
$filter_funct,
|
||||
SORT_NATURAL | SORT_FLAG_CASE,
|
||||
$clauses
|
||||
);
|
||||
}
|
||||
return [
|
||||
'rows' => [],
|
||||
'rowCount' => 0,
|
||||
'total' => 0,
|
||||
'current' => 0
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* delete / drop a specific state by state+creator id
|
||||
*/
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
"""
|
||||
Copyright (c) 2015-2021 Ad Schellevis <ad@opnsense.org>
|
||||
Copyright (c) 2015-2024 Ad Schellevis <ad@opnsense.org>
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
@ -247,18 +247,21 @@ def query_states(rule_label, filter_str):
|
||||
|
||||
|
||||
|
||||
def query_top(rule_label, filter_str):
|
||||
def query_top():
|
||||
addr_parser = AddressParser()
|
||||
result = list()
|
||||
rule_labels = fetch_rule_labels()
|
||||
result = {
|
||||
'details': [],
|
||||
'metadata': {
|
||||
'labels': fetch_rule_labels()
|
||||
}
|
||||
}
|
||||
|
||||
sp = subprocess.run(
|
||||
['/usr/local/sbin/pftop', '-w', '1000', '-b','-v', 'long','9999999999999'],
|
||||
['/usr/local/sbin/pftop', '-w', '1000', '-b','-v', 'long','200000'],
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
filter_net_clauses, filter_clauses = split_filter_clauses(filter_str)
|
||||
|
||||
for rownum, line in enumerate(sp.stdout.strip().split('\n')):
|
||||
parts = line.strip().split()
|
||||
if rownum >= 2 and len(parts) > 5:
|
||||
@ -270,7 +273,7 @@ def query_top(rule_label, filter_str):
|
||||
'dst_addr': addr_parser.split_ip_port(parts[3])['addr'],
|
||||
'dst_port': addr_parser.split_ip_port(parts[3])['port'],
|
||||
'gw_addr': None,
|
||||
'gw_port': None,
|
||||
'gw_port': None
|
||||
}
|
||||
if parts[4].count(':') > 2 or parts[4].count('.') > 2:
|
||||
record['gw_addr'] = addr_parser.split_ip_port(parts[4])['addr']
|
||||
@ -301,12 +304,6 @@ def query_top(rule_label, filter_str):
|
||||
record['bytes'] = 0
|
||||
record['avg'] = int(parts[idx+5]) if parts[idx+5].isdigit() else 0
|
||||
record['rule'] = parts[idx+6]
|
||||
if record['rule'] in rule_labels:
|
||||
record['label'] = rule_labels[record['rule']]['rid']
|
||||
record['descr'] = rule_labels[record['rule']]['descr']
|
||||
else:
|
||||
record['label'] = None
|
||||
record['descr'] = None
|
||||
for timefield in ['age', 'expire']:
|
||||
if ':' in record[timefield]:
|
||||
tmp = record[timefield].split(':')
|
||||
@ -318,35 +315,7 @@ def query_top(rule_label, filter_str):
|
||||
else:
|
||||
record[timefield] = 0
|
||||
|
||||
if rule_label != "" and record['label'].lower().find(rule_label) == -1:
|
||||
# label
|
||||
continue
|
||||
elif filter_clauses or filter_net_clauses:
|
||||
match = False
|
||||
for filter_net in filter_net_clauses:
|
||||
try:
|
||||
match = False
|
||||
for field in ['src_addr', 'dst_addr', 'gw_addr']:
|
||||
port_field = "%s_port" % field[0:3]
|
||||
if record[field] is not None and addr_parser.overlaps(filter_net[0], record[field]):
|
||||
if filter_net[1] is None or filter_net[1] == record[port_field]:
|
||||
match = True
|
||||
if not match:
|
||||
break
|
||||
except:
|
||||
continue
|
||||
if not match:
|
||||
continue
|
||||
|
||||
if filter_clauses:
|
||||
search_line = " ".join(str(item) for item in filter(None, record.values()))
|
||||
for filter_clause in filter_clauses:
|
||||
if search_line.find(filter_clause) == -1:
|
||||
match = False
|
||||
break
|
||||
if not match:
|
||||
continue
|
||||
|
||||
result.append(record)
|
||||
result['details'].append(record)
|
||||
|
||||
return result
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#!/usr/local/bin/python3
|
||||
|
||||
"""
|
||||
Copyright (c) 2021 Ad Schellevis <ad@opnsense.org>
|
||||
Copyright (c) 2021-2024 Ad Schellevis <ad@opnsense.org>
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
@ -26,41 +26,8 @@
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
"""
|
||||
import ujson
|
||||
import argparse
|
||||
from lib.states import query_top
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# parse input arguments
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--filter', help='filter results', default='')
|
||||
parser.add_argument('--limit', help='limit number of results', default='')
|
||||
parser.add_argument('--offset', help='offset results', default='')
|
||||
parser.add_argument('--label', help='label / rule id', default='')
|
||||
parser.add_argument('--sort_by', help='sort by (field asc|desc)', default='')
|
||||
inputargs = parser.parse_args()
|
||||
|
||||
result = {
|
||||
'details': query_top(filter_str=inputargs.filter, rule_label=inputargs.label)
|
||||
}
|
||||
# sort results
|
||||
if inputargs.sort_by.strip() != '' and len(result['details']) > 0:
|
||||
sort_key = inputargs.sort_by.split()[0]
|
||||
sort_desc = inputargs.sort_by.split()[-1] == 'desc'
|
||||
if sort_key in result['details'][0]:
|
||||
if type(result['details'][0][sort_key]) is int:
|
||||
sorter = lambda k: k[sort_key] if sort_key in k else 0
|
||||
else:
|
||||
sorter = lambda k: str(k[sort_key]).lower() if sort_key in k else ''
|
||||
result['details'] = sorted(result['details'], key=sorter, reverse=sort_desc)
|
||||
|
||||
result['total_entries'] = len(result['details'])
|
||||
# apply offset and limit
|
||||
if inputargs.offset.isdigit():
|
||||
result['details'] = result['details'][int(inputargs.offset):]
|
||||
if inputargs.limit.isdigit() and len(result['details']) >= int(inputargs.limit):
|
||||
result['details'] = result['details'][:int(inputargs.limit)]
|
||||
|
||||
result['total'] = len(result['details'])
|
||||
|
||||
print(ujson.dumps(result))
|
||||
print(ujson.dumps(query_top()))
|
||||
|
||||
@ -97,8 +97,8 @@ message:request pf rules
|
||||
|
||||
[diag.top]
|
||||
command:/usr/local/opnsense/scripts/filter/pftop.py
|
||||
parameters: --filter=%s --limit=%s --offset=%s --label=%s --sort_by=%s
|
||||
type:script_output
|
||||
cache_ttl:30
|
||||
message:request pftop statistics
|
||||
|
||||
[diag.info]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user