From fff394c44b8a30bc6070b9a75cf2538c8b25f4f9 Mon Sep 17 00:00:00 2001 From: Ad Schellevis Date: Fri, 20 Mar 2020 19:57:19 +0100 Subject: [PATCH] Shaper: add wrapper to collect different ipfw stats to be combined in the api controller to build later. new call `configctl ipfw stats` There are some assumptions in parsing these stats, ipfw/dummynet man page doesn't seem to provide mich insights on the details delivered by the various "show" commands. for https://github.com/opnsense/core/issues/3994 --- src/opnsense/scripts/shaper/dummynet_stats.py | 41 ++++ src/opnsense/scripts/shaper/lib/__init__.py | 196 ++++++++++++++++++ .../service/conf/actions.d/actions_ipfw.conf | 6 + 3 files changed, 243 insertions(+) create mode 100755 src/opnsense/scripts/shaper/dummynet_stats.py create mode 100644 src/opnsense/scripts/shaper/lib/__init__.py diff --git a/src/opnsense/scripts/shaper/dummynet_stats.py b/src/opnsense/scripts/shaper/dummynet_stats.py new file mode 100755 index 000000000..8ba23344a --- /dev/null +++ b/src/opnsense/scripts/shaper/dummynet_stats.py @@ -0,0 +1,41 @@ +#!/usr/local/bin/python3 + +""" + Copyright (c) 2020 Ad Schellevis + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +""" + +import ujson +from lib import parse_ipfw_pipes, parse_ipfw_queues, parse_ipfw_scheds, parse_ipfw_rules + + +result = dict() +if __name__ == '__main__': + print(ujson.dumps({ + 'pipes': parse_ipfw_pipes(), + 'queues': parse_ipfw_queues(), + 'scheds': parse_ipfw_scheds(), + 'rules': parse_ipfw_rules(), + })) diff --git a/src/opnsense/scripts/shaper/lib/__init__.py b/src/opnsense/scripts/shaper/lib/__init__.py new file mode 100644 index 000000000..6a53cc44d --- /dev/null +++ b/src/opnsense/scripts/shaper/lib/__init__.py @@ -0,0 +1,196 @@ +""" + Copyright (c) 2020 Ad Schellevis + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +""" + + +import subprocess +import ujson +import re + + +def parse_flow(flow_line): + tmp = flow_line.split() + if flow_line.find(':') > 0 and len(tmp) > 8: + # IPv6 layout + return { + 'BKT':tmp[0], + 'Prot':tmp[1], + 'flowid':tmp[2], + 'Source':tmp[3], + 'Destination':tmp[4], + 'pkt':tmp[5], + 'bytes':tmp[6], + 'drop_pkt':tmp[7], + 'drop_bytes':tmp[8] + } + elif len(tmp) > 7: + return { + 'BKT':tmp[0], + 'Prot':tmp[1], + 'Source':tmp[2], + 'Destination':tmp[3], + 'pkt':tmp[4], + 'bytes':tmp[5], + 'drop_pkt':tmp[6], + 'drop_bytes':tmp[7] + } + +def parse_flowset_params(line): + return re.match( + r"q(?P[0-9]*)(?P.*) (?P[0-9]*) flows" + " \((?P[0-9]*) buckets\) sched (?P[0-9]*)" + " weight (?P[0-9]*)" + " lmax (?P[0-9]*)" + " pri (?P[0-9]*)" + "(?P.*)", + line + ) + + +def trim_dict(payload): + for key in payload: + if type(payload[key]) == str: + payload[key] = payload[key].strip() + elif type(payload[key]) == dict: + trim_dict(payload[key]) + return payload + +def parse_ipfw_pipes(): + result = dict() + pipetxt = subprocess.run(['/sbin/ipfw', 'pipe', 'show'], capture_output=True, text=True).stdout.strip() + current_pipe = None + current_pipe_header = False + for line in ("%s\n000000X" % pipetxt).split('\n'): + if len(line) == 0: + continue + if line[0].isdigit(): + if current_pipe: + result[current_pipe['pipe']] = current_pipe + current_pipe_header = False + if line.find('burst') > -1: + current_pipe = { + 'pipe': line[0:5], + 'bw': line[7:line.find('ms') - 5].strip(), + 'delay': line[line.find('ms') - 5: line.find('ms')].strip(), + 'burst': line[line.find('burst ')+6:].strip(), + 'flows': [] + } + elif line[0] == 'q' and current_pipe is not None: + m = parse_flowset_params(line) + if m: + current_pipe['flowset'] = m.groupdict() + elif line.find("RED") > -1 and current_pipe is not None: + current_pipe['flowset']['queue_params'] = line.strip() + elif line.startswith(' sched'): + # + m = re.match( + r" sched (?P[0-9]*) type (?P.*) flags (?P0x[0-9a-fA-F]*)" + " (?P[0-9]*) buckets (?P[0-9]*) active", + line + ) + if m: + current_pipe['scheduler'] = m.groupdict() + elif line.find('__Source') > 0: + current_pipe_header = True + elif current_pipe_header: + flow_stats = parse_flow(line) + if flow_stats: + current_pipe['flows'].append(flow_stats) + + return trim_dict(result) + +def parse_ipfw_queues(): + result = dict() + queuetxt = subprocess.run(['/sbin/ipfw', 'queue', 'show'], capture_output=True, text=True).stdout.strip() + current_queue = None + current_queue_header = False + for line in ("%s\nq000000X" % queuetxt).split('\n'): + if len(line) == 0: + continue + if line[0] == 'q': + m = parse_flowset_params(line) + if current_queue: + result[current_queue['flow_set_nr']] = current_queue + if m: + current_queue = m.groupdict() + current_queue['flows'] = list() + elif line.find('__Source') > 0: + current_queue_header = True + else: + flow_stats = parse_flow(line) + if flow_stats: + current_queue['flows'].append(flow_stats) + + return trim_dict(result) + +def parse_ipfw_scheds(): + result = dict() + schedtxt = subprocess.run(['/sbin/ipfw', 'sched', 'show'], capture_output=True, text=True).stdout.strip() + current_sched = None + for line in ("%s\n000000X" % schedtxt).split('\n'): + if len(line) == 0: + continue + if line[0].isdigit(): + if current_sched: + result[current_sched['pipe']] = current_sched + if line.find('burst') > 0: + current_sched = { + 'pipe': line[0:5] + } + elif line.startswith(' sched'): + m = re.match( + r" sched (?P[0-9]*) type (?P.*) flags (?P0x[0-9a-fA-F]*)" + " (?P[0-9]*) buckets (?P[0-9]*) active", + line + ) + if m: + current_sched.update(m.groupdict()) + elif line.find('Children flowsets') > 0: + current_sched['children'] = line[22:].split() + + return trim_dict(result) + + +def parse_ipfw_rules(): + result = {'queues': list(), 'pipes': list()} + ruletxt = subprocess.run(['/sbin/ipfw', '-aT', 'list'], capture_output=True, text=True).stdout.strip() + for line in ruletxt.split('\n'): + parts = line.split() + if len(parts) > 5 and parts[4] in ['queue', 'pipe']: + rule = { + 'rule': parts[0], + 'pkts': parts[1], + 'bytes': parts[2], + 'accessed': parts[3], + 'rule_uuid': None + } + if line.find('//') > -1: + rule_uuid = line[line.find('//')+3:].strip().split()[0] + if rule_uuid.count('-') == 4: + rule['rule_uuid'] = rule_uuid + result["%ss" % parts[4]].append(rule) + + return result diff --git a/src/opnsense/service/conf/actions.d/actions_ipfw.conf b/src/opnsense/service/conf/actions.d/actions_ipfw.conf index 2d2025fd9..3935361da 100644 --- a/src/opnsense/service/conf/actions.d/actions_ipfw.conf +++ b/src/opnsense/service/conf/actions.d/actions_ipfw.conf @@ -9,3 +9,9 @@ command:/usr/local/etc/rc.ipfw.flush_all parameters: type:script message:flush all ipfw rules + +[stats] +command:/usr/local/opnsense/scripts/shaper/dummynet_stats.py +parameters: +type:script_output +message:request dummynet stats