mirror of
https://github.com/lucaspalomodevelop/core.git
synced 2026-03-16 01:24:38 +00:00
0adece8d3e
When searching large log files for messages that do not frequently occur, there is a large risk of reading all collected lines before returning the first results. In most cases recent items are required, in which case going back for days of logs might not make sense. This commit adds a simple "history" selection which translates into a "valid_from" filter on the log data. When timestamps are not parseable for whatever reason, the filter is ignored. Only small downside is that we do need to translate timestamp again, when needed we could improve performance a bit by storing the original datetime value in NewBaseLogFormat so we don't have to parse it twice.
83 lines
3.7 KiB
Python
Executable File
83 lines
3.7 KiB
Python
Executable File
#!/usr/local/bin/python3
|
|
|
|
"""
|
|
Copyright (c) 2019-2024 Ad Schellevis <ad@opnsense.org>
|
|
Copyright (c) 2024 Deciso B.V.
|
|
All rights reserved.
|
|
|
|
Redistribution and use in source and binary forms, with or without
|
|
modification, are permitted provided that the following conditions are met:
|
|
|
|
1. Redistributions of source code must retain the above copyright notice,
|
|
this list of conditions and the following disclaimer.
|
|
|
|
2. Redistributions in binary form must reproduce the above copyright
|
|
notice, this list of conditions and the following disclaimer in the
|
|
documentation and/or other materials provided with the distribution.
|
|
|
|
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
|
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
|
|
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
|
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
|
|
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
--------------------------------------------------------------------------------------
|
|
|
|
query log files
|
|
"""
|
|
|
|
import os.path
|
|
import ujson
|
|
from dateutil.parser import isoparse
|
|
from log_matcher import LogMatcher
|
|
import argparse
|
|
|
|
if __name__ == '__main__':
|
|
# handle parameters
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--output', help='output type [json/text]', default='json')
|
|
parser.add_argument('--filter', help='filter results', default='')
|
|
parser.add_argument('--limit', help='limit number of results', default='')
|
|
parser.add_argument('--offset', help='begin at row number', default='')
|
|
parser.add_argument('--filename', help='log file name (excluding .log extension)', default='')
|
|
parser.add_argument('--module', help='module', default='core')
|
|
parser.add_argument('--severity', help='comma separated list of severities', default='')
|
|
parser.add_argument('--valid_from', help='oldest data to search for (epoch)', default='')
|
|
inputargs = parser.parse_args()
|
|
|
|
try:
|
|
valid_from = float(inputargs.valid_from)
|
|
except ValueError:
|
|
valid_from = 0
|
|
|
|
result = {'filters': inputargs.filter, 'rows': [], 'total_rows': 0, 'origin': os.path.basename(inputargs.filename)}
|
|
if inputargs.filename != "":
|
|
limit = int(inputargs.limit) if inputargs.limit.isdigit() else 0
|
|
offset = int(inputargs.offset) if inputargs.offset.isdigit() else 0
|
|
log_matcher = LogMatcher(inputargs.filter, inputargs.filename, inputargs.module, inputargs.severity)
|
|
for record in log_matcher.match_records():
|
|
result['total_rows'] += 1
|
|
if (len(result['rows']) < limit or limit == 0) and (result['total_rows'] >= offset):
|
|
if inputargs.output == 'json':
|
|
result['rows'].append(record)
|
|
else:
|
|
print("%(timestamp)s\t%(severity)s\t%(process_name)s\t%(line)s" % record)
|
|
if limit > 0 and result['total_rows'] > offset + limit:
|
|
# do not fetch data until end of file...
|
|
break
|
|
# exit when data found is older than provided valid_from
|
|
try:
|
|
if valid_from and isoparse(record['timestamp']).timestamp() < valid_from:
|
|
break
|
|
except ValueError:
|
|
pass
|
|
|
|
# output results (when json)
|
|
if inputargs.output == 'json':
|
|
print(ujson.dumps(result))
|