Firewall / Diagnostics - time to kill legacy diag_system_pftop.php and replace it with "Sessions", which offers insights into the top sessions on the firewall related to rules and network segments.

This commit is contained in:
Ad Schellevis 2021-07-20 21:53:33 +02:00
parent 75d91570a0
commit e4aa3e4f5e
9 changed files with 335 additions and 175 deletions

View File

@ -202,6 +202,50 @@ class FirewallController extends ApiControllerBase
return [];
}
/**
* query pftop
*/
public function queryPfTopAction()
{
if ($this->request->isPost()) {
$this->sessionClose();
$filter = new Filter([
'query' => function ($value) {
return preg_replace("/[^0-9,a-z,A-Z, ,\/,*,\-,_,.,\#]/", "", $value);
}
]);
$searchPhrase = '';
$ruleId = '';
$sortBy = '';
$itemsPerPage = $this->request->getPost('rowCount', 'int', 9999);
$currentPage = $this->request->getPost('current', 'int', 1);
if ($this->request->getPost('ruleid', 'string', '') != '') {
$ruleId = $filter->sanitize($this->request->getPost('ruleid'), 'query');
}
if ($this->request->getPost('searchPhrase', 'string', '') != '') {
$searchPhrase = $filter->sanitize($this->request->getPost('searchPhrase'), 'query');
}
if ($this->request->has('sort') && is_array($this->request->getPost("sort"))) {
$tmp = array_keys($this->request->getPost("sort"));
$sortBy = $tmp[0] . " " . $this->request->getPost("sort")[$tmp[0]];
}
$response = (new Backend())->configdpRun('filter diag top', [$searchPhrase, $itemsPerPage,
($currentPage - 1) * $itemsPerPage, $ruleId, $sortBy]);
$response = json_decode($response, true);
if ($response != null) {
return [
'rows' => $response['details'],
'rowCount' => count($response['details']),
'total' => $response['total_entries'],
'current' => (int)$currentPage
];
}
}
return [];
}
/**
* delete / drop a specific state by state+creator id
*/

View File

@ -59,4 +59,11 @@ class FirewallController extends IndexController
{
$this->view->pick('OPNsense/Diagnostics/fw_states');
}
/**
* firewall pftop
*/
public function pfTopAction()
{
$this->view->pick('OPNsense/Diagnostics/fw_pftop');
}
}

View File

@ -147,9 +147,10 @@
</patterns>
</page-diagnostics-pf-info>
<page-diagnostics-system-pftop>
<name>Diagnostics: pfTop</name>
<name>Diagnostics: Firewall sessions</name>
<patterns>
<pattern>diag_system_pftop.php*</pattern>
<pattern>ui/diagnostics/firewall/pf_top*</pattern>
<pattern>api/diagnostics/firewall/pf_top*</pattern>
</patterns>
</page-diagnostics-system-pftop>
<page-diagnostics-ping>

View File

@ -188,8 +188,8 @@
<Diagnostics order="500" cssClass="fa fa-medkit fa-fw">
<Aliases url="/ui/firewall/alias_util/"/>
<States url="/ui/diagnostics/firewall/states"/>
<Sessions url="/ui/diagnostics/firewall/pf_top"/>
<pfInfo url="/diag_pf_info.php"/>
<pfTop url="/diag_system_pftop.php"/>
</Diagnostics>
<LogFiles order="400" VisibleName="Log Files" cssClass="fa fa-eye fa-fw">
<Live VisibleName="Live View" url="/ui/diagnostics/firewall/log"/>

View File

@ -0,0 +1,140 @@
{#
# Copyright (c) 2021 Deciso B.V.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#}
<script>
'use strict';
$( document ).ready(function() {
/*************************************************************************************************************
* link grid actions
*************************************************************************************************************/
let grid_pftop = $("#grid-pftop").UIBootgrid(
{ search:'/api/diagnostics/firewall/query_pf_top',
options:{
formatters:{
rule: function (column, row) {
if (row.label !== "") {
return "<a target=\"_blank\" href=\"/firewall_rule_lookup.php?rid=" + row.label + "\">"+row[column.id]+"</a>";
} else {
return row[column.id];
}
},
direction: function (column, row) {
if (row[column.id] == 'out') {
return "<span class=\"fa fa-arrow-left\" title=\"{{lang._('out')}}\" data-toggle=\"tooltip\"></span>";
} else {
return "<span class=\"fa fa-arrow-right\" title=\"{{lang._('in')}}\" data-toggle=\"tooltip\"></span>";
}
},
address: function (column, row) {
if (row[column.id+"_addr"]) {
let addr_txt = row[column.id+"_addr"];
if (addr_txt.includes(":")) {
addr_txt = addr_txt + ":[" + row[column.id+"_port"] + "]";
} else {
addr_txt = addr_txt + ":" + row[column.id+"_port"];
}
return addr_txt;
}
return "";
},
bytes: function(column, row) {
if (!isNaN(row[column.id]) && row[column.id] > 0) {
let fileSizeTypes = ["", "K", "M", "G", "T", "P", "E", "Z", "Y"];
let ndx = Math.floor(Math.log(row[column.id]) / Math.log(1000) );
if (ndx > 0) {
return (row[column.id] / Math.pow(1000, ndx)).toFixed(2) + ' ' + fileSizeTypes[ndx];
} else {
return row[column.id].toFixed(2);
}
} else {
return "";
}
}
},
requestHandler:function(request){
if ($("#ruleid").val() != "") {
request['ruleid'] = $("#ruleid").val();
}
return request;
},
}
}
);
grid_pftop.on('loaded.rs.jquery.bootgrid', function() {
$('[data-toggle="tooltip"]').tooltip();
});
// collect rule id's
ajaxGet("/api/diagnostics/firewall/list_rule_ids", {}, function(data, status){
if (data.items) {
for (let i=0; i < data.items.length ; ++i) {
$("#ruleid").append($("<option/>").val(data.items[i]['id']).text(data.items[i]['descr']));
}
$("#service_status_container").append($("#ruleid"));
$("#ruleid").selectpicker();
$("#ruleid").show();
$("#ruleid").change(function(){
$("#grid-pftop").bootgrid("reload");
});
let init_state = window.location.hash.substr(1);
if (init_state) {
$("#ruleid").val(init_state);
$("#ruleid").change();
}
}
});
});
</script>
<select id="ruleid" data-size="5" data-live-search="true" style="display:none">
<option value="">{{ lang._("Select rule") }}</option>
</select>
<div class="tab-content content-box">
<table id="grid-pftop" class="table table-condensed table-hover table-striped table-responsive" data-editDialog="DialogEdit">
<thead>
<tr>
<th data-column-id="id" data-type="string" data-sortable="false" data-identifier="true" data-visible="false" >{{ lang._('state id') }}</th>
<th data-column-id="direction" data-type="string" data-width="4em" data-formatter="direction">{{ lang._('Dir') }}</th>
<th data-column-id="proto" data-type="string" data-width="6em">{{ lang._('Proto') }}</th>
<th data-column-id="src" data-type="string" data-formatter="address" data-sortable="false">{{ lang._('Source') }}</th>
<th data-column-id="gw" data-type="string" data-formatter="address" data-sortable="false">{{ lang._('Gateway') }}</th>
<th data-column-id="dst" data-type="string" data-formatter="address" data-sortable="false">{{ lang._('Destination') }}</th>
<th data-column-id="state" data-type="string">{{ lang._('State') }}</th>
<th data-column-id="age" data-type="numeric">{{ lang._('Age (sec)') }}</th>
<th data-column-id="expire" data-type="numeric">{{ lang._('Expires (sec)') }}</th>
<th data-column-id="pkts" data-type="numeric" data-formatter="bytes">{{ lang._('Pkts') }}</th>
<th data-column-id="bytes" data-type="numeric" data-formatter="bytes">{{ lang._('Bytes') }}</th>
<th data-column-id="avg" data-type="numeric" data-visible="false">{{ lang._('Avg') }}</th>
<th data-column-id="descr" data-type="string" data-formatter="rule">{{ lang._('Rule') }}</th>
</tr>
</thead>
<tbody>
</tbody>
</table>
</div>

View File

@ -157,3 +157,75 @@ def query_states(rule_label, filter_str):
record['state'] = parts[-1]
return result
def query_top(rule_label, filter_str):
result = list()
rule_labels = fetch_rule_labels()
sp = subprocess.run(['/usr/local/sbin/pftop', '-w', '1000', '-b','-v', 'long','9999999999999'], capture_output=True, text=True)
header = None
try:
filter_network = ipaddress.ip_network(filter_str.strip())
except ValueError:
filter_network = None
for rownum, line in enumerate(sp.stdout.strip().split('\n')):
parts = line.strip().split()
if rownum >= 2 and len(parts) > 5:
record = {
'proto': parts[0],
'dir': parts[1].lower(),
'src_addr': parse_address(parts[2])['addr'],
'src_port': parse_address(parts[2])['port'],
'dst_addr': parse_address(parts[3])['addr'],
'dst_port': parse_address(parts[3])['port'],
'gw_addr': None,
'gw_port': None,
}
if parts[4].count(':') > 2 or parts[4].count('.') > 2:
record['gw_addr'] = parse_address(parts[4])['addr']
record['gw_port'] = parse_address(parts[4])['port']
idx = 5
else:
idx = 4
record['state'] = parts[idx]
record['age'] = parts[idx+1]
record['expire'] = parts[idx+2]
record['pkts'] = int(parts[idx+3])
record['bytes'] = int(parts[idx+4])
record['avg'] = int(parts[idx+5])
record['rule'] = parts[idx+6]
if record['rule'] in rule_labels:
record['label'] = rule_labels[record['rule']]['rid']
record['descr'] = rule_labels[record['rule']]['descr']
else:
record['label'] = None
record['descr'] = None
for timefield in ['age', 'expire']:
tmp = record[timefield].split(':')
record[timefield] = int(tmp[0]) * 3600 + int(tmp[1]) * 60 + int(tmp[2])
search_line = " ".join(str(item) for item in filter(None, record.values()))
if rule_label != "" and record['label'].lower().find(rule_label) == -1:
# label
continue
elif filter_network is not None:
try:
match = False
for field in ['src_addr', 'dst_addr', 'gateway']:
addr = ipaddress.ip_network(record[field])
if field is not None and ipaddress.ip_network(filter_network).overlaps(addr):
match = True
break
if not match:
continue
except:
continue
elif filter_str != "" and search_line.lower().find(filter_str.lower()) == -1:
# apply filter when provided
continue
result.append(record)
return result

View File

@ -0,0 +1,66 @@
#!/usr/local/bin/python3
"""
Copyright (c) 2021 Ad Schellevis <ad@opnsense.org>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
"""
import ujson
import argparse
from lib.states import query_top
if __name__ == '__main__':
# parse input arguments
parser = argparse.ArgumentParser()
parser.add_argument('--filter', help='filter results', default='')
parser.add_argument('--limit', help='limit number of results', default='')
parser.add_argument('--offset', help='offset results', default='')
parser.add_argument('--label', help='label / rule id', default='')
parser.add_argument('--sort_by', help='sort by (field asc|desc)', default='')
inputargs = parser.parse_args()
result = {
'details': query_top(filter_str=inputargs.filter, rule_label=inputargs.label)
}
# sort results
if inputargs.sort_by.strip() != '' and len(result['details']) > 0:
sort_key = inputargs.sort_by.split()[0]
sort_desc = inputargs.sort_by.split()[-1] == 'desc'
if sort_key in result['details'][0]:
if type(result['details'][0][sort_key]) is int:
sorter = lambda k: k[sort_key] if sort_key in k else 0
else:
sorter = lambda k: str(k[sort_key]).lower() if sort_key in k else ''
result['details'] = sorted(result['details'], key=sorter, reverse=sort_desc)
result['total_entries'] = len(result['details'])
# apply offset and limit
if inputargs.offset.isdigit():
result['details'] = result['details'][int(inputargs.offset):]
if inputargs.limit.isdigit() and len(result['details']) >= int(inputargs.limit):
result['details'] = result['details'][:int(inputargs.limit)]
result['total'] = len(result['details'])
print(ujson.dumps(result))

View File

@ -96,8 +96,8 @@ type:script_output
message:request pf rules
[diag.top]
command:/usr/local/sbin/pftop
parameters: -w 200 -b -o %s -v %s %s
command:/usr/local/opnsense/scripts/filter/pftop.py
parameters: --filter=%s --limit=%s --offset=%s --label=%s --sort_by=%s
type:script_output
message:request pftop statistics

View File

@ -1,170 +0,0 @@
<?php
/*
* Copyright (C) 2014 Deciso B.V.
* Copyright (C) 2008-2009 Scott Ullrich <sullrich@gmail.com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
require_once("guiconfig.inc");
$sorttypes = array('age', 'bytes', 'dest', 'dport', 'exp', 'none', 'peak', 'pkt', 'rate', 'size', 'sport', 'src');
$viewtypes = array('default', 'label', 'long', 'rules', 'size', 'speed', 'state', 'time');
$numstates = array('50', '100', '200', '500', '1000', '99999999999');
if ($_SERVER['REQUEST_METHOD'] === 'POST') {
// fetch only valid input data (items from above lists)
$viewtype = 'default';
$numstate = '200';
$sorttype ='bytes';
if (isset($_POST['viewtype']) && in_array($_POST['viewtype'], $viewtypes)) {
$viewtype = $_POST['viewtype'];
}
if (isset($_POST['states']) && in_array($_POST['states'], $numstates)) {
$numstate = $_POST['states'];
}
if (isset($_POST['sorttype']) && in_array($_POST['sorttype'], $sorttypes)) {
$sorttype = $_POST['sorttype'];
}
// fetch pftop data
echo configdp_run('filter diag top', array($sorttype, $viewtype, $numstate));
exit;
}
include("head.inc");
?>
<body>
<?php include("fbegin.inc"); ?>
<script>
$( document ).ready(function() {
/**
* fetch pftop data from backend
*/
function getpftopactivity() {
$.ajax(
'/diag_system_pftop.php',
{
type: 'post',
data: {'getactivity':'yes'
,'sorttype':$('#sorttype').val()
,'viewtype':$('#viewtype').val()
,'states':$('#states').val()
},
complete: function(transport) {
$('#pftopactivitydiv').html('<pre>' + transport.responseText + '<\/pre>');
setTimeout(getpftopactivity, 2500);
}
});
}
$("#viewtype").change(function() {
var selected = $("#viewtype option:selected").val();
switch(selected) {
case "rules":
$(".show_opt").addClass("hidden");
break;
default:
$(".show_opt").removeClass("hidden");
}
});
// toggle initial viewtype select
$("#viewtype").change();
// start initial fetch
getpftopactivity();
});
</script>
<section class="page-content-main">
<div class="container-fluid">
<div class="row">
<div class="table-responsive">
<form method="post">
<table class="table table-striped">
<thead>
<tr>
<th><?=gettext("View type:"); ?></th>
<th class="show_opt"><?=gettext("Sort type:"); ?></th>
<th class="show_opt"><?=gettext("Number of States:"); ?></th>
</tr>
</thead>
<tbody>
<tr>
<td>
<select name='viewtype' id='viewtype' class="selectpicker" data-width="auto" data-live-search="true">
<option value='default' selected="selected"><?=gettext("Default");?></option>
<option value='label'><?=gettext("Label");?></option>
<option value='long'><?=gettext("Long");?></option>
<option value='rules'><?=gettext("Rules");?></option>
<option value='size'><?=gettext("Size");?></option>
<option value='speed'><?=gettext("Speed");?></option>
<option value='state'><?=gettext("State");?></option>
<option value='time'><?=gettext("Time");?></option>
</select>
</td>
<td class="show_opt">
<div>
<select name='sorttype' id='sorttype' class="selectpicker" data-width="auto" data-live-search="true">
<option value='age'><?=gettext("Age");?></option>
<option value='bytes'><?=gettext("Bytes");?></option>
<option value='dest'><?=gettext("Destination Address");?></option>
<option value='dport'><?=gettext("Destination Port");?></option>
<option value='exp'><?=gettext("Expiry");?></option>
<option value='none'><?=gettext("None");?></option>
<option value='peak'><?=gettext("Peak");?></option>
<option value='pkt'><?=gettext("Packet");?></option>
<option value='rate'><?=gettext("Rate");?></option>
<option value='size'><?=gettext("Size");?></option>
<option value='sport'><?=gettext("Source Port");?></option>
<option value='src'><?=gettext("Source Address");?></option>
</select>
</div>
</td>
<td class="show_opt">
<div id='statesdiv'>
<select name='states' id='states' class="selectpicker" data-width="auto" data-live-search="true">
<option value='50'>50</option>
<option value='100'>100</option>
<option value='200' selected="selected">200</option>
<option value='500'>500</option>
<option value='1000'>1000</option>
<option value='99999999999'><?= gettext('all') ?></option>
</select>
</div>
</td>
</tr>
</tbody>
</table>
</form>
<section class="col-xs-12">
<div id="pftopactivitydiv"><?=gettext("Gathering pfTOP activity, please wait...");?></div>
</section>
</div>
</div>
</div>
</section>
<?php include("foot.inc");