diff --git a/plist b/plist index 5c75bc9e2..83c2a00cb 100644 --- a/plist +++ b/plist @@ -1519,7 +1519,6 @@ /usr/local/opnsense/site-python/daemonize.py /usr/local/opnsense/site-python/duckdb_helper.py /usr/local/opnsense/site-python/log_helper.py -/usr/local/opnsense/site-python/params.py /usr/local/opnsense/site-python/sqlite3_helper.py /usr/local/opnsense/site-python/tls_helper.py /usr/local/opnsense/site-python/watchers/__init__.py diff --git a/src/opnsense/scripts/netflow/export_details.py b/src/opnsense/scripts/netflow/export_details.py index 740ea2e85..5bfd99460 100755 --- a/src/opnsense/scripts/netflow/export_details.py +++ b/src/opnsense/scripts/netflow/export_details.py @@ -1,7 +1,7 @@ #!/usr/local/bin/python3 """ - Copyright (c) 2016 Ad Schellevis + Copyright (c) 2016-2025 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -28,46 +28,34 @@ -------------------------------------------------------------------------------------- fetch detailed data from provider for specified timeserie """ +import argparse import time import datetime import pytz -import os import sys -sys.path.insert(0, "/usr/local/opnsense/site-python") import lib.aggregates -import params -app_params = {'start_time': '0', - 'end_time': '1461251783', - 'resolution': '300', - 'provider': 'FlowSourceAddrTotals' - } -params.update_params(app_params) +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--provider', default='FlowInterfaceTotals') + parser.add_argument('--resolution', type=int, required=True) + parser.add_argument('--start_time', type=int, required=True) + parser.add_argument('--end_time', type=int, required=True) + cmd_args = parser.parse_args() -# handle input parameters -valid_params = False -if app_params['start_time'].isdigit(): - start_time = int(app_params['start_time']) - if app_params['end_time'].isdigit(): - end_time = int(app_params['end_time']) - if app_params['resolution'].isdigit(): - resolution = int(app_params['resolution']) - valid_params = True - -if valid_params: # calculate time offset between localtime and utc now_timestamp = time.time() time_offset = datetime.datetime.fromtimestamp(now_timestamp) - datetime.datetime.utcfromtimestamp(now_timestamp) for agg_class in lib.aggregates.get_aggregators(): - if app_params['provider'] == agg_class.__name__: - if resolution in agg_class.resolutions(): + if cmd_args.provider == agg_class.__name__: + if cmd_args.resolution in agg_class.resolutions(): # found provider and resolution, start spooling data - obj = agg_class(resolution) + obj = agg_class(cmd_args.resolution) rownum=0 column_names = dict() - for record in obj.get_data(start_time, end_time): + for record in obj.get_data(cmd_args.start_time, cmd_args.end_time): if rownum == 0: column_names = list(record.keys()) # dump heading @@ -88,14 +76,3 @@ if valid_params: line.append(record[item]) print (','.join(line)) rownum += 1 -else: - print ('missing parameters :') - tmp = list() - for key in app_params: - tmp.append('/%s %s' % (key, app_params[key])) - print (' %s %s'%(sys.argv[0], ' '.join(tmp))) - print ('') - print (' resolution : sample rate in seconds') - print (' start_time : start time (seconds since epoch)') - print (' end_time : end timestamp (seconds since epoch)') - print (' provider : data provider classname') diff --git a/src/opnsense/scripts/netflow/flowd_aggregate.py b/src/opnsense/scripts/netflow/flowd_aggregate.py index 82a572027..106e64c33 100755 --- a/src/opnsense/scripts/netflow/flowd_aggregate.py +++ b/src/opnsense/scripts/netflow/flowd_aggregate.py @@ -1,6 +1,6 @@ #!/usr/local/bin/python3 """ - Copyright (c) 2016-2018 Ad Schellevis + Copyright (c) 2016-2025 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -192,15 +192,12 @@ class Main(object): if __name__ == '__main__': # parse arguments and load config parser = argparse.ArgumentParser() - parser.add_argument('--config', help='configuration yaml', default=None) parser.add_argument('--console', dest='console', help='run in console', action='store_true') parser.add_argument('--profile', dest='profile', help='enable profiler', action='store_true') parser.add_argument('--repair', dest='repair', help='init repair', action='store_true') cmd_args = parser.parse_args() - Main.set_config( - load_config(cmd_args.config) - ) + Main.set_config(load_config()) from sqlite3_helper import check_and_repair if cmd_args.console: diff --git a/src/opnsense/scripts/netflow/flowd_aggregate_metadata.py b/src/opnsense/scripts/netflow/flowd_aggregate_metadata.py index 711f00337..3248db7f5 100755 --- a/src/opnsense/scripts/netflow/flowd_aggregate_metadata.py +++ b/src/opnsense/scripts/netflow/flowd_aggregate_metadata.py @@ -1,7 +1,7 @@ #!/usr/local/bin/python3 """ - Copyright (c) 2016-2019 Ad Schellevis + Copyright (c) 2016-2025 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -38,11 +38,10 @@ import lib.aggregates result = dict() parser = argparse.ArgumentParser() -parser.add_argument('--config', '--config', help='configuration yaml', default=None) parser.add_argument('format', help='output format [text (default)|json]') cmd_args = parser.parse_args() -configuration = load_config(cmd_args.config) +configuration = load_config() # load global metadata metadata = AggMetadata(database_dir=configuration.database_dir) diff --git a/src/opnsense/scripts/netflow/get_timeseries.py b/src/opnsense/scripts/netflow/get_timeseries.py index 5cacd325c..0e07c635b 100755 --- a/src/opnsense/scripts/netflow/get_timeseries.py +++ b/src/opnsense/scripts/netflow/get_timeseries.py @@ -1,7 +1,7 @@ #!/usr/local/bin/python3 """ - Copyright (c) 2016-2019 Ad Schellevis + Copyright (c) 2016-2025 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -39,7 +39,6 @@ import lib.aggregates if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--config', '--config', help='configuration yaml', default=None) parser.add_argument('--provider', default='FlowInterfaceTotals') parser.add_argument('--resolution', type=int, required=True) parser.add_argument('--start_time', type=int, required=True) @@ -47,7 +46,7 @@ if __name__ == '__main__': parser.add_argument('--key_fields', required=True) parser.add_argument('--sample', default='') cmd_args = parser.parse_args() - configuration = load_config(cmd_args.config) + configuration = load_config() timeseries = dict() if cmd_args.sample == '': diff --git a/src/opnsense/scripts/netflow/get_top_usage.py b/src/opnsense/scripts/netflow/get_top_usage.py index 5095dfbd7..a75a21ed0 100755 --- a/src/opnsense/scripts/netflow/get_top_usage.py +++ b/src/opnsense/scripts/netflow/get_top_usage.py @@ -1,7 +1,7 @@ #!/usr/local/bin/python3 """ - Copyright (c) 2016-2019 Ad Schellevis + Copyright (c) 2016-2025 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -37,7 +37,6 @@ from lib import load_config if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--config', '--config', help='configuration yaml', default=None) parser.add_argument('--provider', default='FlowInterfaceTotals') parser.add_argument('--start_time', type=int, required=True) parser.add_argument('--end_time', type=int, required=True) @@ -46,7 +45,7 @@ if __name__ == '__main__': parser.add_argument('--filter', default='') parser.add_argument('--max_hits', type=int, required=True) cmd_args = parser.parse_args() - configuration = load_config(cmd_args.config) + configuration = load_config() result = dict() for agg_class in lib.aggregates.get_aggregators(): diff --git a/src/opnsense/scripts/netflow/lib/__init__.py b/src/opnsense/scripts/netflow/lib/__init__.py index edaf70d3b..8d914e17b 100755 --- a/src/opnsense/scripts/netflow/lib/__init__.py +++ b/src/opnsense/scripts/netflow/lib/__init__.py @@ -1,5 +1,5 @@ """ - Copyright (c) 2018 Ad Schellevis + Copyright (c) 2018-2025 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -26,17 +26,11 @@ import sys -def load_config(config_yaml=None): +def load_config(): """ setup configuration object - :param config_yaml: - :return: + :return: dict """ - if config_yaml: - import yaml - cnf_input = yaml.load(open(config_yaml, 'r')) - else: - cnf_input = dict() - + cnf_input = dict() result = Config(**cnf_input) sys.path.insert(0, result.library_path) diff --git a/src/opnsense/scripts/netflow/lib/aggregates/__init__.py b/src/opnsense/scripts/netflow/lib/aggregates/__init__.py index da4c6d707..70e8c5a00 100755 --- a/src/opnsense/scripts/netflow/lib/aggregates/__init__.py +++ b/src/opnsense/scripts/netflow/lib/aggregates/__init__.py @@ -1,5 +1,5 @@ """ - Copyright (c) 2016-2018 Ad Schellevis + Copyright (c) 2016-2025 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -308,10 +308,10 @@ class BaseFlowAggregator(object): query_params['end_time'] = self._parse_timestamp(end_time) if data_filters: for data_filter in data_filters.split(','): - tmp = data_filter.split('=')[0].strip() + tmp = data_filter.split('=', 1)[0].strip() if tmp in self.agg_fields and data_filter.find('=') > -1: filter_fields.append(tmp) - query_params[tmp] = '='.join(data_filter.split('=')[1:]) + query_params[tmp] = data_filter.split('=', 1)[1] if len(select_fields) > 0: # construct sql query to filter and select data diff --git a/src/opnsense/service/conf/actions.d/actions_netflow.conf b/src/opnsense/service/conf/actions.d/actions_netflow.conf index 567e6ac76..8d3a549e4 100644 --- a/src/opnsense/service/conf/actions.d/actions_netflow.conf +++ b/src/opnsense/service/conf/actions.d/actions_netflow.conf @@ -95,7 +95,7 @@ message:request netflow data aggregator top usage for %s [aggregate.export] command:/usr/local/opnsense/scripts/netflow/export_details.py -parameters:/provider %s /start_time %s /end_time %s /resolution %s +parameters:--provider %s --start_time %s --end_time %s --resolution %s type:script_output message:export netflow data aggregator details for %s diff --git a/src/opnsense/site-python/params.py b/src/opnsense/site-python/params.py deleted file mode 100644 index 18c49fb9f..000000000 --- a/src/opnsense/site-python/params.py +++ /dev/null @@ -1,46 +0,0 @@ -""" - Copyright (c) 2015-2016 Ad Schellevis - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, - OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. -""" - -import sys - - -def update_params(parameters): - """ update predefined parameters with given list from shell (as switches) - for example /a valA /b valB - converts to - {'a':'valA','b':'valB'} - (assuming parameters contains both a and b) - :param parameters: parameter dictionary - :return: - """ - cmd = None - for arg in sys.argv[1:]: - if cmd is None: - cmd = arg[1:] - else: - if cmd in parameters and arg.strip() != '': - parameters[cmd] = arg.strip() - cmd = None