From 0adece8d3e165acc0ba3bb2e1d8f0e6593dd8c41 Mon Sep 17 00:00:00 2001 From: Ad Schellevis Date: Sat, 24 Aug 2024 20:55:12 +0200 Subject: [PATCH] System/Logging - add log search time constraint (valid_from) to limit searches when only a few lines match. When searching large log files for messages that do not frequently occur, there is a large risk of reading all collected lines before returning the first results. In most cases recent items are required, in which case going back for days of logs might not make sense. This commit adds a simple "history" selection which translates into a "valid_from" filter on the log data. When timestamps are not parseable for whatever reason, the filter is ignored. Only small downside is that we do need to translate timestamp again, when needed we could improve performance a bit by storing the original datetime value in NewBaseLogFormat so we don't have to parse it twice. --- .../Diagnostics/Api/LogController.php | 5 ++-- .../app/views/OPNsense/Diagnostics/log.volt | 26 ++++++++++++++++--- src/opnsense/scripts/syslog/queryLog.py | 15 ++++++++++- .../conf/actions.d/actions_system.conf | 4 +-- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/LogController.php b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/LogController.php index 03abb278b..5094a0960 100644 --- a/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/LogController.php +++ b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/LogController.php @@ -74,7 +74,8 @@ class LogController extends ApiControllerBase $searchPhrase, $module, $scope, - $severities + $severities, + $this->request->getPost('validFrom', null, '0') ]); $result = json_decode($response, true); if ($result != null) { @@ -119,7 +120,7 @@ class LogController extends ApiControllerBase return $this->configdStream( 'system diag log_live', - [$offset, $searchPhrase, $module, $scope, $severities], + [$offset, $searchPhrase, $module, $scope, $severities, $this->request->get('validFrom', null, '0')], [ 'Content-Type: text/event-stream', 'Cache-Control: no-cache' diff --git a/src/opnsense/mvc/app/views/OPNsense/Diagnostics/log.volt b/src/opnsense/mvc/app/views/OPNsense/Diagnostics/log.volt index dce66df6e..199510bb8 100644 --- a/src/opnsense/mvc/app/views/OPNsense/Diagnostics/log.volt +++ b/src/opnsense/mvc/app/views/OPNsense/Diagnostics/log.volt @@ -43,6 +43,7 @@ } else { s_filter_val = localStorage.getItem('log_severity_{{module}}_{{scope}}') ? localStorage.getItem('log_severity_{{module}}_{{scope}}').split(',') : s_filter_val; } + $("#validFrom_filter").val(localStorage.getItem('log_validFrom_filter_{{module}}_{{scope}}') ? localStorage.getItem('log_validFrom_filter_{{module}}_{{scope}}') : 'day'); } switch_mode(s_filter_val); @@ -71,14 +72,24 @@ // get selected severities or severities below or equal to selected request['severity'] = filter_exact ? selectedSeverity : severities.slice(0,severities.indexOf(selectedSeverity) + 1); } + let time_offsets = { + 'day': 60*60*24, + 'week': 7*60*60*24, + 'month': 31*60*60*24, + } + if ($("#validFrom_filter").val().length > 0 && time_offsets[$("#validFrom_filter").val()]) { + let now = Date.now() / 1000; + request['validFrom'] = now - time_offsets[$("#validFrom_filter").val()]; + } return request; }, }, search:'/api/diagnostics/log/{{module}}/{{scope}}' }); - $("#severity_filter").change(function(){ + $(".filter_act").change(function(){ if (window.localStorage) { localStorage.setItem('log_severity_{{module}}_{{scope}}', $("#severity_filter").val()); + localStorage.setItem('log_validFrom_filter_{{module}}_{{scope}}', $("#validFrom_filter").val()); } $('#grid-log').bootgrid('reload'); }); @@ -144,7 +155,8 @@ updateServiceControlUI('{{service}}'); // move filter into action header - $("#severity_filter_container").detach().prependTo('#grid-log-header > .row > .actionBar > .actions'); + $("#filter_container").detach().prependTo('#grid-log-header > .row > .actionBar > .actions'); + $(".filter_act").tooltip(); function switch_mode(value) { @@ -198,8 +210,8 @@
diff --git a/src/opnsense/scripts/syslog/queryLog.py b/src/opnsense/scripts/syslog/queryLog.py index fe2bcb6dd..e4f62f895 100755 --- a/src/opnsense/scripts/syslog/queryLog.py +++ b/src/opnsense/scripts/syslog/queryLog.py @@ -1,7 +1,7 @@ #!/usr/local/bin/python3 """ - Copyright (c) 2019-2020 Ad Schellevis + Copyright (c) 2019-2024 Ad Schellevis Copyright (c) 2024 Deciso B.V. All rights reserved. @@ -33,6 +33,7 @@ import os.path import ujson +from dateutil.parser import isoparse from log_matcher import LogMatcher import argparse @@ -46,8 +47,14 @@ if __name__ == '__main__': parser.add_argument('--filename', help='log file name (excluding .log extension)', default='') parser.add_argument('--module', help='module', default='core') parser.add_argument('--severity', help='comma separated list of severities', default='') + parser.add_argument('--valid_from', help='oldest data to search for (epoch)', default='') inputargs = parser.parse_args() + try: + valid_from = float(inputargs.valid_from) + except ValueError: + valid_from = 0 + result = {'filters': inputargs.filter, 'rows': [], 'total_rows': 0, 'origin': os.path.basename(inputargs.filename)} if inputargs.filename != "": limit = int(inputargs.limit) if inputargs.limit.isdigit() else 0 @@ -63,6 +70,12 @@ if __name__ == '__main__': if limit > 0 and result['total_rows'] > offset + limit: # do not fetch data until end of file... break + # exit when data found is older than provided valid_from + try: + if valid_from and isoparse(record['timestamp']).timestamp() < valid_from: + break + except ValueError: + pass # output results (when json) if inputargs.output == 'json': diff --git a/src/opnsense/service/conf/actions.d/actions_system.conf b/src/opnsense/service/conf/actions.d/actions_system.conf index b9bd10a18..2543a4ba3 100644 --- a/src/opnsense/service/conf/actions.d/actions_system.conf +++ b/src/opnsense/service/conf/actions.d/actions_system.conf @@ -6,13 +6,13 @@ message:Show system activity [diag.log] command:/usr/local/opnsense/scripts/syslog/queryLog.py -parameters:--limit %s --offset %s --filter %s --module %s --filename %s --severity %s +parameters:--limit %s --offset %s --filter %s --module %s --filename %s --severity %s --valid_from %s type:script_output message:Show log [diag.log_stream] command:/usr/local/opnsense/scripts/syslog/queryLog.py -parameters:--limit %s --offset %s --filter %s --module %s --filename %s --severity %s --output text +parameters:--limit %s --offset %s --filter %s --module %s --filename %s --severity %s --valid_from %s --output text type:stream_output message:Stream log