lucaspalomodevelop/core
1
0
Fork 0
mirror of https://github.com/lucaspalomodevelop/core.git synced 2026-03-15 17:14:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

System/Logging - add log search time constraint (valid_from) to limit searches when only a few lines match.

Browse Source 
When searching large log files for messages that do not frequently occur, there is a large risk of reading all collected lines before returning the first results.
In most cases recent items are required, in which case going back for days of logs might not make sense.

This commit adds a simple "history" selection which translates into a "valid_from" filter on the log data. When timestamps are not parseable for whatever reason, the filter is ignored.

Only small downside is that we do need to translate timestamp again, when needed we could improve performance a bit by storing the original datetime value in NewBaseLogFormat so we don't have to parse it twice.
This commit is contained in:
Ad Schellevis 2024-08-24 20:55:12 +02:00
parent 798170b612
commit 0adece8d3e
4 changed files with 41 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/LogController.php
View File

@ -74,7 +74,8 @@ class LogController extends ApiControllerBase
$searchPhrase,
$module,
$scope,
$severities
$severities,
$this->request->getPost('validFrom', null, '0')
]);
$result = json_decode($response, true);
if ($result != null) {
@ -119,7 +120,7 @@ class LogController extends ApiControllerBase
return $this->configdStream(
'system diag log_live',
[$offset, $searchPhrase, $module, $scope, $severities],
[$offset, $searchPhrase, $module, $scope, $severities, $this->request->get('validFrom', null, '0')],
[
'Content-Type: text/event-stream',
'Cache-Control: no-cache'

26
src/opnsense/mvc/app/views/OPNsense/Diagnostics/log.volt
View File

@ -43,6 +43,7 @@
} else {
s_filter_val = localStorage.getItem('log_severity_{{module}}_{{scope}}') ? localStorage.getItem('log_severity_{{module}}_{{scope}}').split(',') : s_filter_val;
}
$("#validFrom_filter").val(localStorage.getItem('log_validFrom_filter_{{module}}_{{scope}}') ? localStorage.getItem('log_validFrom_filter_{{module}}_{{scope}}') : 'day');
}
switch_mode(s_filter_val);
@ -71,14 +72,24 @@
// get selected severities or severities below or equal to selected
request['severity'] = filter_exact ? selectedSeverity : severities.slice(0,severities.indexOf(selectedSeverity) + 1);
}
let time_offsets = {
'day': 60*60*24,
'week': 7*60*60*24,
'month': 31*60*60*24,
}
if ($("#validFrom_filter").val().length > 0 && time_offsets[$("#validFrom_filter").val()]) {
let now = Date.now() / 1000;
request['validFrom'] = now - time_offsets[$("#validFrom_filter").val()];
}
return request;
},
},
search:'/api/diagnostics/log/{{module}}/{{scope}}'
});
$("#severity_filter").change(function(){
$(".filter_act").change(function(){
if (window.localStorage) {
localStorage.setItem('log_severity_{{module}}_{{scope}}', $("#severity_filter").val());
localStorage.setItem('log_validFrom_filter_{{module}}_{{scope}}', $("#validFrom_filter").val());
}
$('#grid-log').bootgrid('reload');
});
@ -144,7 +155,8 @@
updateServiceControlUI('{{service}}');
// move filter into action header
$("#severity_filter_container").detach().prependTo('#grid-log-header > .row > .actionBar > .actions');
$("#filter_container").detach().prependTo('#grid-log-header > .row > .actionBar > .actions');
$(".filter_act").tooltip();
function switch_mode(value) {
@ -198,8 +210,8 @@
<div class="col-sm-12">
<div class="hidden">
<!-- filter per type container -->
<div id="severity_filter_container" class="btn-group">
<select id="severity_filter" data-title="{{ lang._('Severity') }}" data-width="200px">
<div id="filter_container" class="btn-group">
<select id="severity_filter" data-title="{{ lang._('Severity') }}" class="filter_act" data-width="200px">
<option value="Emergency">{{ lang._('Emergency') }}</option>
<option value="Alert">{{ lang._('Alert') }}</option>
<option value="Critical">{{ lang._('Critical') }}</option>
@ -209,6 +221,12 @@
<option value="Informational">{{ lang._('Informational') }}</option>
<option value="Debug">{{ lang._('Debug') }}</option>
</select>
<select id="validFrom_filter" data-title="{{ lang._('History') }}" class="filter_act selectpicker" data-width="200px">
<option selected="selected" value="day">{{ lang._('1 Day') }}</option>
<option value="week">{{ lang._('1 Week') }}</option>
<option value="month">{{ lang._('1 Month') }}</option>
<option value="all">{{ lang._('All') }}</option>
</select>
</div>
</div>
<table id="grid-log" class="table table-condensed table-hover table-striped table-responsive">

15
src/opnsense/scripts/syslog/queryLog.py
View File

@ -1,7 +1,7 @@
#!/usr/local/bin/python3
"""
Copyright (c) 2019-2020 Ad Schellevis <ad@opnsense.org>
Copyright (c) 2019-2024 Ad Schellevis <ad@opnsense.org>
Copyright (c) 2024 Deciso B.V.
All rights reserved.
@ -33,6 +33,7 @@
import os.path
import ujson
from dateutil.parser import isoparse
from log_matcher import LogMatcher
import argparse
@ -46,8 +47,14 @@ if __name__ == '__main__':
parser.add_argument('--filename', help='log file name (excluding .log extension)', default='')
parser.add_argument('--module', help='module', default='core')
parser.add_argument('--severity', help='comma separated list of severities', default='')
parser.add_argument('--valid_from', help='oldest data to search for (epoch)', default='')
inputargs = parser.parse_args()
try:
valid_from = float(inputargs.valid_from)
except ValueError:
valid_from = 0
result = {'filters': inputargs.filter, 'rows': [], 'total_rows': 0, 'origin': os.path.basename(inputargs.filename)}
if inputargs.filename != "":
limit = int(inputargs.limit) if inputargs.limit.isdigit() else 0
@ -63,6 +70,12 @@ if __name__ == '__main__':
if limit > 0 and result['total_rows'] > offset + limit:
# do not fetch data until end of file...
break
# exit when data found is older than provided valid_from
try:
if valid_from and isoparse(record['timestamp']).timestamp() < valid_from:
break
except ValueError:
pass
# output results (when json)
if inputargs.output == 'json':

4
src/opnsense/service/conf/actions.d/actions_system.conf
View File

@ -6,13 +6,13 @@ message:Show system activity
[diag.log]
command:/usr/local/opnsense/scripts/syslog/queryLog.py
parameters:--limit %s --offset %s --filter %s --module %s --filename %s --severity %s
parameters:--limit %s --offset %s --filter %s --module %s --filename %s --severity %s --valid_from %s
type:script_output
message:Show log
[diag.log_stream]
command:/usr/local/opnsense/scripts/syslog/queryLog.py
parameters:--limit %s --offset %s --filter %s --module %s --filename %s --severity %s --output text
parameters:--limit %s --offset %s --filter %s --module %s --filename %s --severity %s --valid_from %s --output text
type:stream_output
message:Stream log