lucaspalomodevelop/core
1
0
Fork 0
mirror of https://github.com/lucaspalomodevelop/core.git synced 2026-03-19 11:04:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

configd: extend streaming support for blocking processes

Browse Source 
This allows for hooking into the EventSource mechanism on the client side,
enabling server-sent events without busylooping on the backend.
This will reduce stateless network chatter and eliminates the need for
polling and many other benefits.

Continuation of f25e1214dc,
which disables buffering on the webserver side. This change in particular also
removes implicit buffering on the configd side.

As an example, the polling of CPU usage is included with a backend script here.
Granted, this could easily be replaced by `iostat -w 1 cpu | egrep -v "tty|tin" --line-buffered`,
but the client will eventually need some form of per-event formatting which is already
being handled in this example. When implementing these types of scripts, make sure
that all output that encapsulates a single event is flushed at all times to prevent
OS buffering. A new controller (without any consumers) is also implemented to showcase
the passthrough mechanism on the controller side.
This commit is contained in:
Stephan de Wit 2024-03-22 10:29:44 +01:00
parent d27eb4261c
commit 89d4af5a52
6 changed files with 144 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
plist
View File

@ -263,6 +263,7 @@
/usr/local/opnsense/mvc/app/controllers/OPNsense/DHCRelay/forms/dialogRelay.xml
/usr/local/opnsense/mvc/app/controllers/OPNsense/Diagnostics/ActivityController.php
/usr/local/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/ActivityController.php
/usr/local/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/CpuUsageController.php
/usr/local/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/DnsController.php
/usr/local/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/DnsDiagnosticsController.php
/usr/local/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/FirewallController.php
@ -1127,6 +1128,7 @@
/usr/local/opnsense/scripts/syslog/queryLog.py
/usr/local/opnsense/scripts/system/activity.py
/usr/local/opnsense/scripts/system/certctl.py
/usr/local/opnsense/scripts/system/cpu.py
/usr/local/opnsense/scripts/system/nameservers.php
/usr/local/opnsense/scripts/system/remote_backup.php
/usr/local/opnsense/scripts/system/rfc5246_cipher_suites.csv

52
src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/CpuUsageController.php Normal file
View File

@ -0,0 +1,52 @@
<?php
/*
* Copyright (C) 2024 Deciso B.V.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
namespace OPNsense\Diagnostics\Api;
use OPNsense\Base\ApiControllerBase;
use OPNsense\Core\Config;
use OPNsense\Core\Backend;
/**
* Class CpuUsage
* @package OPNsense\Diagnostics
*/
class CpuUsageController extends ApiControllerBase
{
public function streamAction()
{
return $this->configdStream(
'system cpu stream',
['1'],
[
'Content-Type: text/event-stream',
'Cache-Control: no-cache'
]
);
}
}

65
src/opnsense/scripts/system/cpu.py Executable file
View File

@ -0,0 +1,65 @@
#!/usr/local/bin/python3
"""
Copyright (c) 2024 Deciso B.V.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
--------------------------------------------------------------------------------------
streams cpu usage
"""
import subprocess
import select
import argparse
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--interval', help='poll interval', default='1')
inputargs = parser.parse_args()
process = subprocess.Popen(
['iostat', '-w', inputargs.interval, 'cpu'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
bufsize=0
)
read_fds = [process.stdout]
while True:
readable, _, _ = select.select(read_fds, [], [])
for fd in readable:
data = fd.readline()
if data:
output = data.decode().strip()
if not (output.startswith("tty") or output.startswith("tin")):
print(f"event: message\ndata: {output}\n\n", flush=True)
else:
read_fds.remove(fd)
if process.poll() is not None:
break
process.stdout.close()
process.stderr.close()

6
src/opnsense/service/conf/actions.d/actions_system.conf
View File

@ -102,3 +102,9 @@ command:/usr/local/opnsense/scripts/system/trust_configure.php
parameters:
type:script
message:configure trust
[cpu.stream]
command:/usr/local/opnsense/scripts/system/cpu.py
parameters:--interval %s
type:stream_output
message:Stream CPU stats

5
src/opnsense/service/configd_ctl.py
View File

@ -67,9 +67,12 @@ def exec_config_cmd(exec_command):
yield line
else:
break
except KeyboardInterrupt:
# intentional
pass
except:
syslog_error('error in configd communication \n%s'%traceback.format_exc())
print ('error in configd communication %s, see syslog for details', file=sys.stderr)
print ('error in configd communication, see syslog for details', file=sys.stderr)
finally:
sock.close()

18
src/opnsense/service/modules/actions/stream_output.py
View File

@ -27,6 +27,8 @@
import traceback
import selectors
import subprocess
import os
import signal
from .. import syslog_error, syslog_info
from .base import BaseAction
@ -50,20 +52,25 @@ class Action(BaseAction):
env=self.config_environment,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
stderr=subprocess.PIPE,
bufsize=0,
preexec_fn=os.setsid
)
selector = selectors.DefaultSelector()
selector.register(process.stdout, selectors.EVENT_READ, stdout_reader)
try:
while process.poll() is None:
timeout = True
for key, mask in selector.select(1):
timeout = False
callback = key.data
callback(key.fileobj, mask)
try:
callback(key.fileobj, mask)
except BrokenPipeError:
timeout = True
break
if timeout:
# when timeout has reached, check if the other end is still connected using getpeername()
# kill process when nobody is waiting for an answer
@ -72,6 +79,11 @@ class Action(BaseAction):
except OSError:
syslog_info('[%s] Script action terminated by other end' % message_uuid)
process.kill()
# kill child processes as well
try:
os.killpg(os.getpgid(process.pid), signal.SIGKILL)
except ProcessLookupError:
pass
return None
return_code = process.wait()