diff --git a/src/opnsense/scripts/netflow/export_details.py b/src/opnsense/scripts/netflow/export_details.py
new file mode 100755
index 000000000..20f79ca14
--- /dev/null
+++ b/src/opnsense/scripts/netflow/export_details.py
@@ -0,0 +1,97 @@
+#!/usr/local/bin/python2.7
+
+"""
+ Copyright (c) 2016 Ad Schellevis
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+ OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.
+
+ --------------------------------------------------------------------------------------
+ fetch detailed data from provider for specified timeserie
+"""
+import time
+import datetime
+import os
+import sys
+import ujson
+sys.path.insert(0, "/usr/local/opnsense/site-python")
+from lib.parse import parse_flow
+from lib.aggregate import BaseFlowAggregator
+import lib.aggregates
+import params
+
+
+app_params = {'start_time': '0',
+ 'end_time': '1461251783',
+ 'resolution': '300',
+ 'provider': 'FlowSourceAddrTotals'
+ }
+params.update_params(app_params)
+
+# handle input parameters
+valid_params = False
+if app_params['start_time'].isdigit():
+ start_time = int(app_params['start_time'])
+ if app_params['end_time'].isdigit():
+ end_time = int(app_params['end_time'])
+ if app_params['resolution'].isdigit():
+ resolution = int(app_params['resolution'])
+ valid_params = True
+
+if valid_params:
+ for agg_class in lib.aggregates.get_aggregators():
+ if app_params['provider'] == agg_class.__name__:
+ if resolution in agg_class.resolutions():
+ # found provider and resolution, start spooling data
+ obj = agg_class(resolution)
+ rownum=0
+ column_names = dict()
+ for record in obj.get_data(start_time, end_time):
+ if rownum == 0:
+ column_names = record.keys()
+ # dump heading
+ print (','.join(column_names))
+ line = list()
+ for item in column_names:
+ if not record[item]:
+ line.append("")
+ if type(record[item]) == datetime.datetime:
+ line.append('%s+00:00'%record[item].strftime('%Y/%m/%d %H:%M:%S'))
+ elif type(record[item]) == float:
+ line.append('%.4f' % record[item])
+ elif type(record[item]) == int:
+ line.append('%d' % record[item])
+ else:
+ line.append(record[item])
+ print (','.join(line))
+ rownum += 1
+else:
+ print ('missing parameters :')
+ tmp = list()
+ for key in app_params:
+ tmp.append('/%s %s' % (key, app_params[key]))
+ print (' %s %s'%(sys.argv[0], ' '.join(tmp)))
+ print ('')
+ print (' resolution : sample rate in seconds')
+ print (' start_time : start time (seconds since epoch)')
+ print (' end_time : end timestamp (seconds since epoch)')
+ print (' provider : data provider classname')
diff --git a/src/opnsense/scripts/netflow/lib/aggregate.py b/src/opnsense/scripts/netflow/lib/aggregate.py
index 4978cb0f8..11d675f54 100644
--- a/src/opnsense/scripts/netflow/lib/aggregate.py
+++ b/src/opnsense/scripts/netflow/lib/aggregate.py
@@ -31,6 +31,33 @@ import os
import datetime
import sqlite3
+def convert_timestamp(val):
+ """ convert timestamps from string (internal sqlite type) or seconds since epoch
+ """
+ if val.find('-') > -1:
+ # formatted date/time
+ if val.find(" ") > -1:
+ datepart, timepart = val.split(" ")
+ else:
+ datepart = val
+ timepart = "0:0:0,0"
+ year, month, day = map(int, datepart.split("-"))
+ timepart_full = timepart.split(".")
+ hours, minutes, seconds = map(int, timepart_full[0].split(":"))
+ if len(timepart_full) == 2:
+ microseconds = int('{:0<6.6}'.format(timepart_full[1].decode()))
+ else:
+ microseconds = 0
+
+ val = datetime.datetime(year, month, day, hours, minutes, seconds, microseconds)
+ else:
+ # timestamp stored as seconds since epoch, convert to utc
+ val = datetime.datetime.utcfromtimestamp(float(val))
+
+ return val
+
+sqlite3.register_converter('timestamp', convert_timestamp)
+
class AggMetadata(object):
""" store some metadata needed to keep track of parse progress
"""
@@ -384,3 +411,32 @@ class BaseFlowAggregator(object):
cur.close()
return result
+
+ def get_data(self, start_time, end_time):
+ """ get detail data
+ :param start_time: start timestamp
+ :param end_time: end timestamp
+ :return: iterator
+ """
+ query_params = {}
+ query_params['start_time'] = self._parse_timestamp((int(start_time/self.resolution))*self.resolution)
+ query_params['end_time'] = self._parse_timestamp(end_time)
+ sql_select = 'select mtime start_time, '
+ sql_select += '%s, octets, packets, last_seen as "last_seen [timestamp]" \n' % ','.join(self.agg_fields)
+ sql_select += 'from timeserie \n'
+ sql_select += 'where mtime >= :start_time and mtime < :end_time\n'
+ cur = self._db_connection.cursor()
+ cur.execute(sql_select, query_params)
+
+ # fetch all data, to a max of [max_hits] rows.
+ field_names = (map(lambda x:x[0], cur.description))
+ while True:
+ record = cur.fetchone()
+ if record is None:
+ break
+ else:
+ result_record=dict()
+ for field_indx in range(len(field_names)):
+ if len(record) > field_indx:
+ result_record[field_names[field_indx]] = record[field_indx]
+ yield result_record
diff --git a/src/opnsense/service/conf/actions.d/actions_netflow.conf b/src/opnsense/service/conf/actions.d/actions_netflow.conf
index 922950d25..c4ab8e223 100644
--- a/src/opnsense/service/conf/actions.d/actions_netflow.conf
+++ b/src/opnsense/service/conf/actions.d/actions_netflow.conf
@@ -85,3 +85,9 @@ command:/usr/local/opnsense/scripts/netflow/get_top_usage.py
parameters:/provider %s /start_time %s /end_time %s /key_fields %s /value_field %s /filter %s /max_hits %s
type:script_output
message:request netflow data aggregator top usage for %s
+
+[aggregate.export]
+command:/usr/local/opnsense/scripts/netflow/export_details.py
+parameters:/provider %s /start_time %s /end_time %s /resolution %s
+type:script_output
+message:export netflow data aggregator details for %s