lucaspalomodevelop/core
1
0
Fork 0
mirror of https://github.com/lucaspalomodevelop/core.git synced 2026-03-14 08:34:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reporting / Health - refactor rrd data retrieval and simplify usage

Browse Source 
Most of this code is quite old and originates from the beginning of our project. At the time it seemed to be problematic to render the full rrd stats in a d3 graph, which required the "resolution" option for faster page loading. It looks like we can safely remove this toggle and ditch quite some code in the process. There's still room for improvements in the html/javascript part, but that's probably for another day.

This commit also simplifies the api usage as unused parameters are being removed from the callers (from, to, ..)
This commit is contained in:
Ad Schellevis 2023-10-05 15:51:37 +02:00
parent e5e8d003bd
commit 972a7d60bf
3 changed files with 232 additions and 733 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

522
src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/SystemhealthController.php
View File

@ -1,6 +1,7 @@
<?php
/*
* Copyright (C) 2023 Deciso B.V.
* Copyright (C) 2015 Jos Schellevis <jos@opnsense.org>
* All rights reserved.
*
@ -38,420 +39,6 @@ use OPNsense\Core\Config;
*/
class SystemhealthController extends ApiControllerBase
{
/**
* Return full archive information
* @param \SimpleXMLElement $xml rrd data xml
* @return array info set, metadata
*/
private function getDataSetInfo($xml)
{
$info = array();
if (isset($xml)) {
$step = intval($xml->step);
$lastUpdate = intval($xml->lastupdate);
foreach ($xml->rra as $key => $value) {
$step_size = (int)$value->pdp_per_row * $step;
$first = floor(($lastUpdate / $step_size)) * $step_size -
($step_size * (count($value->database->children()) - 1));
$last = floor(($lastUpdate / $step_size)) * $step_size;
$firstValue_rowNumber = (int)$this->findFirstValue($value);
$firstValue_timestamp = (int)$first + ((int)$firstValue_rowNumber * $step_size);
array_push($info, [
"step" => $step,
"pdp_per_row" => (int)$value->pdp_per_row,
"rowCount" => $this->countRows($value),
"first_timestamp" => (int)$first,
"last_timestamp" => (int)$last,
"firstValue_rowNumber" => $firstValue_rowNumber,
"firstValue_timestamp" => $firstValue_timestamp,
"available_rows" => ($this->countRows($value) - $firstValue_rowNumber),
"full_step" => ($step * (int)$value->pdp_per_row),
"recorded_time" => ($step * (int)$value->pdp_per_row) *
($this->countRows($value) - $firstValue_rowNumber)
]);
}
}
return ($info);
}
/**
* Returns row number of first row with values other than 'NaN'
* @param \SimpleXMLElement $data rrd data xml
* @return int rownumber
*/
private function findFirstValue($data)
{
$rowNumber = 0;
foreach ($data->database->row as $item => $row) {
foreach ($row as $rowKey => $rowVal) {
if (trim($rowVal) != "NaN") {
break 2;
}
}
}
return $rowNumber;
}
/**
* Return total number of rows in rra
* @param \SimpleXMLElement $data rrd data xml
* @return int total number of rows
*/
private function countRows($data)
{
$rowCount = 0;
foreach ($data->database->row as $item => $row) {
$rowCount++;
}
return $rowCount;
}
/**
* internal: retrieve selections within range (0-0=full range) and limit number of datapoints (max_values)
* @param array $rra_info dataset information
* @param int $from_timestamp from
* @param int $to_timestamp to
* @param int $max_values approx. max number of values
* @return array
*/
private function getSelection($rra_info, $from_timestamp, $to_timestamp, $max_values)
{
if ($from_timestamp == 0 && $to_timestamp == 0) {
$from_timestamp = $this->getMaxRange($rra_info)["oldest_timestamp"];
$to_timestamp = $this->getMaxRange($rra_info)["newest_timestamp"];
}
$max_values = ($max_values <= 0) ? 1 : $max_values;
$archives = array();
// find archive match
foreach ($rra_info as $key => $value) {
if (
$from_timestamp >= $value['firstValue_timestamp'] && $to_timestamp <= ($value['last_timestamp'] +
$value['full_step'])
) {
// calculate number of rows in set
$rowCount = ($to_timestamp - $from_timestamp) / $value['full_step'] + 1;
// factor to be used to compress the data.
// example if 2 then 2 values will be used to calculate one data point, minimum 1 (don't condense).
$condense_factor = round($rowCount / $max_values) < 1 ? 1 : round($rowCount / $max_values);
// actual number of rows after compressing/condensing the dataSet
$condensed_rowCount = (int)($rowCount / $condense_factor);
// count the number if rra's (sets), deduct 1 as we need the counter to start at 0
$last_rra_key = count($rra_info) - 1;
if ($from_timestamp == 0 && $to_timestamp == 0) { // add detail when selected
array_push($archives, [
"key" => $key,
"condensed_rowCount" => $condensed_rowCount,
"condense_by" => (int)$condense_factor,
"type" => "detail"
]);
} else { // add condensed detail
array_push($archives, [
"key" => $key,
"condensed_rowCount" => (int)($condensed_rowCount / ($rra_info[$last_rra_key]["pdp_per_row"] /
$value["pdp_per_row"])),
"condense_by" => (int)$condense_factor * ($rra_info[$last_rra_key]["pdp_per_row"] /
$value["pdp_per_row"]),
"type" => "detail"
]);
}
// search for last dataSet with actual values, used to exclude sets that do not contain data
for ($count = $last_rra_key; $count > 0; $count--) {
if ($rra_info[$count]["available_rows"] > 0) {
// Found last rra set with values
$last_rra_key = $count;
break;
}
}
// dynamic (condensed) values for full overview to detail level
$overview = round($rra_info[$last_rra_key]["available_rows"] / (int)$max_values);
if ($overview != 0) {
$condensed_rowCount = (int)($rra_info[$last_rra_key]["available_rows"] / $overview);
} else {
$condensed_rowCount = 0;
}
array_push($archives, [
"key" => $last_rra_key,
"condensed_rowCount" => $condensed_rowCount,
"condense_by" => (int)$overview,
"type" => "overview"
]);
break;
}
}
return (["from" => $from_timestamp, "to" => $to_timestamp,
"full_range" => ($from_timestamp == 0 && $to_timestamp == 0),
"data" => $archives]);
}
/**
* internal: get full available range
* @param array $rra_info
* @return array
*/
private function getMaxRange($rra_info)
{
// count the number if rra's (sets), deduct 1 as we need the counter to start at 0
$last_rra_key = count($rra_info) - 1;
for ($count = $last_rra_key; $count > 0; $count--) {
if ($rra_info[$count]["available_rows"] > 0) {
// Found last rra set with values
$last_rra_key = $count;
break;
}
}
if (isset($rra_info[0])) {
$last = $rra_info[0]["firstValue_timestamp"];
$first = $rra_info[$last_rra_key]["firstValue_timestamp"] + $rra_info[$last_rra_key]["recorded_time"] -
$rra_info[$last_rra_key]["full_step"];
} else {
$first = 0;
$last = 0;
}
return ["newest_timestamp" => $first, "oldest_timestamp" => $last];
}
/**
* translate rrd data to usable format for d3 charts
* @param array $data
* @param boolean $applyInverse inverse selection (multiply -1)
* @param array $field_units mapping for descriptive field names
* @return array
*/
private function translateD3($data, $applyInverse, $field_units)
{
$d3_data = array();
$from_timestamp = 0;
$to_timestamp = 0;
foreach ($data['archive'] as $row => $rowValues) {
$timestamp = $rowValues['timestamp'] * 1000; // javascript works with milliseconds
foreach ($data['columns'] as $key => $value) {
$name = $value['name'];
$value = $rowValues['condensed_values'][$key];
if (!isset($d3_data[$key])) {
$d3_data[$key] = [];
$d3_data[$key]["area"] = true;
if (isset($field_units[$name])) {
$d3_data[$key]["key"] = $name . " " . $field_units[$name];
} else {
$d3_data[$key]["key"] = $name;
}
$d3_data[$key]["values"] = [];
}
if ($value == "NaN") {
// If first or the last NaN value in series then add a value of 0 for presentation purposes
$nan = false;
if (
isset($data['archive'][$row - 1]['condensed_values'][$key]) &&
(string)$data['archive'][$row - 1]['condensed_values'][$key] != "NaN"
) {
// Translate NaN to 0 as d3chart can't render NaN - (first NaN item before value)
$value = 0;
} elseif (
isset($data['archive'][$row + 1]['condensed_values'][$key]) &&
(string)$data['archive'][$row + 1]['condensed_values'][$key] != "NaN"
) {
$value = 0; // Translate NaN to 0 as d3chart can't render NaN - (last NaN item before value)
} else {
$nan = true; // suppress NaN item as we already drawn a line to 0
}
} else {
$nan = false; // Not a NaN value, so add to list
}
if ($value != "NaN" && $applyInverse) {
if ($key % 2 != 0) {
$value = $value * -1;
}
}
if (!$nan) {
if ($from_timestamp == 0 || $timestamp < $from_timestamp) {
$from_timestamp = $timestamp; // Actual from_timestamp after condensing and cleaning data
}
if ($to_timestamp == 0 || $timestamp > $to_timestamp) {
$to_timestamp = $timestamp; // Actual to_timestamp after condensing and cleaning data
}
array_push($d3_data[$key]["values"], [$timestamp, $value]);
}
}
}
// Sort value sets based on timestamp
foreach ($d3_data as $key => $value) {
usort($value["values"], array($this, "orderByTimestampASC"));
$d3_data[$key]["values"] = $value["values"];
}
return [
"stepSize" => $data['condensed_step'],
"from_timestamp" => $from_timestamp,
"to_timestamp" => $to_timestamp,
"count" => isset($d3_data[0]) ? count($d3_data[0]['values']) : 0,
"data" => $d3_data
];
}
/**
* retrieve rrd data
* @param array $xml
* @param array $selection
* @return array
*/
private function getCondensedArchive($xml, $selection)
{
$key_counter = 0;
$info = $this->getDataSetInfo($xml);
$count_values = 0;
$condensed_row_values = array();
$condensed_archive = array();
$condensed_step = 0;
$skip_nan = false;
$selected_archives = $selection["data"];
foreach ($xml->rra as $key => $value) {
$calculation_type = trim($value->cf);
foreach ($value->database as $db_key => $db_value) {
foreach ($selected_archives as $archKey => $archValue) {
if ($archValue['key'] == $key_counter) {
$rowCount = 0;
$condense_counter = 0;
$condense = $archValue['condense_by'];
foreach ($db_value as $rowKey => $rowValues) {
if ($rowCount >= $info[$key_counter]['firstValue_rowNumber']) {
$timestamp = $info[$key_counter]['first_timestamp'] +
($rowCount * $info[$key_counter]['step'] * $info[$key_counter]['pdp_per_row']);
if (
($timestamp >= $selection["from"] && $timestamp <= $selection["to"] &&
$archValue["type"] == "detail") || ($archValue["type"] == "overview" &&
$timestamp <= $selection["from"]) || ($archValue["type"] == "overview" &&
$timestamp >= $selection["to"])
) {
$condense_counter++;
// Find smallest step in focus area = detail
if ($archValue['type'] == "detail" && $selection["full_range"] == false) {
// Set new calculated step size
$condensed_step = ($info[$key_counter]['full_step'] * $condense);
} else {
if ($selection["full_range"] == true && $archValue['type'] == "overview") {
$condensed_step = ($info[$key_counter]['full_step'] * $condense);
}
}
$column_counter = 0;
if (!isset($condensed_row_values[$count_values])) {
$condensed_row_values[$count_values] = [];
}
foreach ($rowValues->v as $columnKey => $columnValue) {
if (!isset($condensed_row_values[$count_values][$column_counter])) {
$condensed_row_values[$count_values][$column_counter] = 0;
}
if (trim($columnValue) == "NaN") {
// skip processing the rest of the values as this set has a NaN value
$skip_nan = true;
$condensed_row_values[$count_values][$column_counter] = "NaN";
} elseif ($skip_nan == false) {
if ($archValue["type"] == "overview") {
// overwrite this values and skip averaging, looks better for overview
$condensed_row_values[$count_values][$column_counter] =
((float)$columnValue);
} elseif ($calculation_type == "AVERAGE") {
// For AVERAGE always add the values
$condensed_row_values[$count_values][$column_counter] +=
(float)$columnValue;
} elseif ($calculation_type == "MINIMUM" || $condense_counter == 1) {
// For MINIMUM update value if smaller one found or first
if (
$condensed_row_values[$count_values][$column_counter] >
(float)$columnValue
) {
$condensed_row_values[$count_values][$column_counter] =
(float)$columnValue;
}
} elseif ($calculation_type == "MAXIMUM" || $condense_counter == 1) {
// For MAXIMUM update value if higher one found or first
if (
$condensed_row_values[$count_values][$column_counter] <
(float)$columnValue
) {
$condensed_row_values[$count_values][$column_counter] =
(float)$columnValue;
}
}
}
$column_counter++;
}
if ($condense_counter == $condense) {
foreach ($condensed_row_values[$count_values] as $crvKey => $crValue) {
if (
$condensed_row_values[$count_values][$crvKey] != "NaN" &&
$calculation_type == "AVERAGE" && $archValue["type"] != "overview"
) {
// For AVERAGE we need to calculate it,
// dividing by the total number of values collected
$condensed_row_values[$count_values][$crvKey] =
(float)$condensed_row_values[$count_values][$crvKey] / $condense;
}
}
$skip_nan = false;
if ($info[$key_counter]['available_rows'] > 0) {
array_push($condensed_archive, [
"timestamp" => $timestamp - ($info[$key_counter]['step'] *
$info[$key_counter]['pdp_per_row']),
"condensed_values" => $condensed_row_values[$count_values]
]);
}
$count_values++;
$condense_counter = 0;
}
}
}
$rowCount++;
}
}
}
}
$key_counter++;
}
// get value information to include in set
$column_data = array();
foreach ($xml->ds as $key => $value) {
array_push($column_data, ["name" => trim($value->name), "type" => trim($value->type)]);
}
return ["condensed_step" => $condensed_step, "columns" => $column_data, "archive" => $condensed_archive];
}
/**
* Custom Compare for usort
* @param $a
* @param $b
* @return mixed
*/
private function orderByTimestampASC($a, $b)
{
return $a[0] - $b[0];
}
/**
* retrieve descriptive details of rrd
* @param string $rrd rrd category - item
@ -460,9 +47,8 @@ class SystemhealthController extends ApiControllerBase
private function getRRDdetails($rrd)
{
# Source of data: xml fields of corresponding .xml metadata
$result = array();
$backend = new Backend();
$response = $backend->configdRun('health list');
$result = [];
$response = (new Backend())->configdRun('health list');
$healthList = json_decode($response, true);
// search by topic and name, return array with filename
if (is_array($healthList)) {
@ -490,14 +76,9 @@ class SystemhealthController extends ApiControllerBase
public function getRRDlistAction()
{
# Source of data: filelisting of /var/db/rrd/*.rrd
$result = array();
$backend = new Backend();
$response = $backend->configdRun('health list');
$healthList = json_decode($response, true);
$result = ['data' => []];
$healthList = json_decode((new Backend())->configdRun('health list'), true);
$interfaces = $this->getInterfacesAction();
$result['data'] = array();
if (is_array($healthList)) {
foreach ($healthList as $healthItem => $details) {
if (!array_key_exists($details['topic'], $result['data'])) {
@ -534,81 +115,42 @@ class SystemhealthController extends ApiControllerBase
/**
* retrieve SystemHealth Data (previously called RRD Graphs)
* @param string $rrd
* @param int $from
* @param int $to
* @param int $max_values
* @param bool $inverse
* @param int $detail
* @return array
*/
public function getSystemHealthAction(
$rrd = "",
$from = 0,
$to = 0,
$max_values = 120,
$inverse = false,
$detail = -1
) {
/**
* $rrd = rrd filename without extension
* $from = from timestamp (0=min)
* $to = to timestamp (0=max)
* $max_values = limit datapoint as close as possible to this number (or twice if detail (zoom) + overview )
* $inverse = Inverse every odd row (multiply by -1)
* $detail = limits processing of dataSets to max given (-1 = all ; 1 = 0,1 ; 2 = 0,1,2 ; etc)
*/
public function getSystemHealthAction($rrd = "", $inverse = 0, $detail = -1) {
$rrd_details = $this->getRRDdetails($rrd)["data"];
$xml = false;
if ($rrd_details['filename'] != "") {
$backend = new Backend();
$response = $backend->configdpRun('health fetch', [$rrd_details['filename']]);
if ($response != null) {
$xml = @simplexml_load_string($response);
}
}
if ($xml !== false) {
// we only use the average databases in any RRD, remove the rest to avoid strange behaviour.
for ($count = count($xml->rra) - 1; $count >= 0; $count--) {
if (trim((string)$xml->rra[$count]->cf) != "AVERAGE") {
unset($xml->rra[$count]);
$response = (new Backend())->configdpRun('health fetch', [$rrd_details['filename']]);
$response = json_decode($response ?? '', true);
if (!empty($response)) {
$response['set'] = ['count' => 0, 'data' => [], 'step_size' => 0];
if (isset($response['sets'][$detail])) {
$records = $response['sets'][$detail]['ds'];
$response['set']['count'] = count($records);
$response['set']['step_size'] = $response['sets'][$detail]['step_size'];
foreach ($records as $key => $record) {
$record['area'] = true;
if ($inverse && $key % 2 != 0) {
foreach ($record['values'] as &$value) {
$value[1] = $value[1]*-1;
}
}
$response['set']['data'][] = $record;
}
}
$data_sets_full = $this->getDataSetInfo($xml); // get dataSet information to include in answer
if ($inverse == 'true') {
$inverse = true;
for ($i=0; $i < count($response['sets']); $i++) {
unset($response['sets'][$detail]['ds']);
}
if (!empty($rrd_details["title"])) {
$response['title'] = $rrd_details["title"] . " | " . ucfirst($rrd_details['itemName']);
} else {
$inverse = false;
$response['title'] = ucfirst($rrd_details['itemName']);
}
// The zoom (timespan) level determines the number of datasets to
// use in the equation. All the irrelevant sets are removed here.
if ((int)$detail >= 0) {
for ($count = count($xml->rra) - 1; $count > $detail; $count--) {
unset($xml->rra[$count]);
}
}
// determine available dataSets within range and how to handle them
$selected_archives = $this->getSelection($this->getDataSetInfo($xml), $from, $to, $max_values);
// get condensed dataSets and translate them to d3 usable data
$result = $this->translateD3(
$this->getCondensedArchive($xml, $selected_archives),
$inverse,
$rrd_details["field_units"]
);
return ["sets" => $data_sets_full,
"d3" => $result,
"title" => $rrd_details["title"] != "" ?
$rrd_details["title"] . " | " . ucfirst($rrd_details['itemName']) :
ucfirst($rrd_details['itemName']),
"y-axis_label" => $rrd_details["y-axis_label"]
]; // return details and d3 data
$response['y-axis_label'] = $rrd_details["y-axis_label"];
return $response;
} else {
return ["sets" => [], "d3" => [], "title" => "error", "y-axis_label" => ""];
return ["sets" => [], "set" => [], "title" => "error", "y-axis_label" => ""];
}
}
@ -619,7 +161,7 @@ class SystemhealthController extends ApiControllerBase
public function getInterfacesAction()
{
// collect interface names
$intfmap = array();
$intfmap = [];
$config = Config::getInstance()->object();
if ($config->interfaces->count() > 0) {
foreach ($config->interfaces->children() as $key => $node) {

368
src/opnsense/mvc/app/views/OPNsense/Diagnostics/health.volt
View File

@ -1,5 +1,6 @@
<!--
/*
* Copyright (C) 2023 Deciso B.V.
* Copyright (C) 2015 Jos Schellevis <jos@opnsense.org>
* All rights reserved.
*
@ -53,17 +54,14 @@
<script>
var chart;
var data = [];
var fetching_data = true;
var current_selection_from = 0;
var current_selection_to = 0;
var disabled = [];
var resizeTimer;
var current_detail = 0;
var csvData = [];
var zoom_buttons;
var rrd="";
let chart;
let data = [];
let disabled = [];
let resizeTimer;
let current_detail = 0;
let csvData = [];
let zoom_buttons;
let rrd="";
// create our chart
@ -114,42 +112,15 @@
return chart;
});
// Some options have changed, check and fetch data
function UpdateOptions() {
window.onresize = null; // clear any pending resize events
var inverse = false;
var detail = 0;
var resolution = 120;
if ($('input:radio[name=inverse]:checked').val() == 1) {
inverse = true;
}
detail = $('input:radio[name=detail]:checked').val();
resolution = $('input:radio[name=resolution]:checked').val();
if (detail != current_detail) {
chart.brushExtent([0, 0]);
getdata(rrd, 0, 0, resolution, detail);
current_detail = detail;
} else {
getdata(rrd, current_selection_from, current_selection_to, resolution, detail);
current_detail = detail;
}
}
function getRRDlist() {
ajaxGet("/api/diagnostics/systemhealth/getRRDlist/", {}, function (data, status) {
if (status == "success") {
if (data.data.length == 0 ) {
$(".page-content-head").removeClass("hidden");
$('#info_tab').toggleClass('active');
return;
}
var category;
var tabs="";
var subitem="";
var active_category=Object.keys(data["data"])[0];
var active_subitem=data["data"][active_category][0];
var rrd_name="";
let category;
let tabs = "";
let subitem = "";
let active_category = Object.keys(data["data"])[0];
let active_subitem = data["data"][active_category][0];
let rrd_name = "";
for ( category in data["data"]) {
if (category == active_category) {
tabs += '<li role="presentation" class="dropdown active">';
@ -157,7 +128,7 @@
tabs += '<li role="presentation" class="dropdown">';
}
subitem=data["data"][category][0]; // first sub item
subitem = data["data"][category][0]; // first sub item
rrd_name = subitem + '-' + category;
// create dropdown menu
@ -166,19 +137,16 @@
tabs+='</a>';
tabs+='<a data-toggle="tab" onclick="$(\'#'+rrd_name+'\').click();" class="visible-lg-inline-block visible-md-inline-block visible-xs-inline-block visible-sm-inline-block" style="border-right:0px;"><b>'+category[0].toUpperCase() + category.slice(1)+'</b></a>';
tabs+='<ul class="dropdown-menu" role="menu">';
rrd_name="";
// add subtabs
for (var count=0; count<data["data"][category].length;++count ) {
for (let count=0; count<data["data"][category].length;++count ) {
subitem=data["data"][category][count];
rrd_name = subitem + '-' + category;
if (subitem==active_subitem && category==active_category) {
tabs += '<li class="active"><a data-toggle="tab" class="rrd-item" onclick="getdata(\''+rrd_name+'\',0,0,120,0);" id="'+rrd_name+'">' + subitem[0].toUpperCase() + subitem.slice(1) + '</a></li>';
rrd=rrd_name;
getdata(rrd_name,0,0,120,false,0); // load initial data
tabs += '<li class="active"><a data-toggle="tab" class="rrd_item" id="'+rrd_name+'">' + subitem[0].toUpperCase() + subitem.slice(1) + '</a></li>';
} else {
tabs += '<li><a data-toggle="tab" class="rrd-item" onclick="getdata(\''+rrd_name+'\',0,0,120,0);" id="'+rrd_name+'">' + subitem[0].toUpperCase() + subitem.slice(1) + '</a></li>';
tabs += '<li><a data-toggle="tab" class="rrd_item" id="'+rrd_name+'">' + subitem[0].toUpperCase() + subitem.slice(1) + '</a></li>';
}
}
tabs+='</ul>';
@ -186,72 +154,45 @@
}
$('#maintabs').html(tabs);
$('#tab_1').toggleClass('active');
// map interface descriptions
$(".rrd-item").each(function(){
var rrd_item = $(this);
var rrd_item_name = $(this).attr('id').split('-')[0].toLowerCase();
$(".rrd_item").each(function(){
let rrd_item = $(this);
let rrd_item_name = $(this).attr('id').split('-')[0].toLowerCase();
$.map(data['interfaces'], function(value, key){
if (key.toLowerCase() == rrd_item_name) {
rrd_item.html(value['descr']);
}
});
});
} else {
alert("Error while fetching RRD list : "+status);
$(".rrd_item").click(function(){
// switch between rrd graphs
$('#zoom').empty();
disabled = []; // clear disabled stream data
chart.brushExtent([0, 0]); // clear focus area
getdata($(this).attr('id'));
});
$(".update_options").change(function(){
window.onresize = null; // clear any pending resize events
let rrd = $(".dropdown-menu > li.active > a").attr('id');
if ($(this).attr('id') == 'zoom') {
chart.brushExtent([0, 0]); // clear focus area
}
getdata(rrd);
});
$("#"+active_subitem+"-"+active_category).click();
}
});
}
function getdata(rrd_name, from, to, maxitems, detail) {
function getdata(rrd_name) {
if (zoom_buttons===undefined) {
zoom_buttons="";
}
let from = 0;
let to = 0;
let maxitems = 120;
// Set defaults if not specified
if (rrd_name === undefined) {
rrd_name = rrd;
disabled = []; // clear disabled stream data
} else {
if ( rrd_name!=rrd ) {
rrd = rrd_name; // set global rrd name to current rrd
disabled = []; // clear disabled stream data
zoom_buttons=""; // clear zoom_buttons
chart.brushExtent([0, 0]); // clear focus area
$('#res0').parent().click(); // reset resolution
}
}
if (from === undefined) {
from = 0;
}
if (to === undefined) {
to = 0;
}
if (maxitems === undefined) {
maxitems = 120;
}
let inverse = false;
if ($('input:radio[name=inverse]:checked').val() == 1) {
inverse = true;
}
if (detail === undefined) {
detail = 0;
}
// Remember selected area
current_selection_from = from;
current_selection_to = to;
// Flag to know when we are fetching data
fetching_data = true;
// Used to set render the zoom/detail buttons
//zoom_buttons = "";
let detail = $('input:radio[name=detail]:checked').val() ?? '0';
let inverse = $('input:radio[name=inverse]:checked').val() == 1 ? '1' : '0';
// array used for cvs export option
csvData = [];
@ -265,12 +206,11 @@
// info bar - show loading info bar while refreshing data
$('#loading').show();
// API call to request data
ajaxGet("/api/diagnostics/systemhealth/getSystemHealth/" + rrd_name + "/" + String(from) + "/" + String(to) + "/" + String(maxitems) + "/" + String(inverse) + "/" + String(detail), {}, function (data, status) {
ajaxGet("/api/diagnostics/systemhealth/getSystemHealth/" + rrd_name + "/" + inverse + "/" + detail, {}, function (data, status) {
if (status == "success") {
var stepsize = data["d3"]["stepSize"];
var scale = "{{ lang._('seconds') }}";
var dtformat = '%m/%d %H:%M';
var visible_time=to-from;
let stepsize = data["set"]["step_size"];
let scale = "{{ lang._('seconds') }}";
let dtformat = '%m/%d %H:%M';
// set defaults based on stepsize
if (stepsize >= 86400) {
@ -287,53 +227,57 @@
dtformat = '%H:%M';
}
// if we have a focus area then change the x-scale to reflect current view
if (visible_time >= (86400*7)) { // one week
dtformat = '\'%y w%U%';
} else if (visible_time >= (3600*48)) { // 48 hours
dtformat = '\'%y d%j%';
} else if (visible_time >= (60*maxitems)) { // max minutes
dtformat = '%H:%M';
}
// Add zoomlevel buttons/options
if ($('input:radio[name=detail]:checked').val() == undefined || zoom_buttons==="") {
if ($('input:radio[name=detail]:checked').val() == undefined) {
$('#zoom').html("<b>No data available</b>");
for (let setcount = 0; setcount < data["sets"].length; ++setcount) {
const recordedtime = data["sets"][setcount]["recorded_time"];
const set_stepsize = data["sets"][setcount]["step_size"];
let detail_text = '';
// Find out what text matches best
if (recordedtime >= 31536000) {
detail_text = Math.floor(recordedtime / 31536000).toString() + " {{ lang._('Year(s)') }}";
} else if (recordedtime >= 259200) {
detail_text = Math.floor(recordedtime / 86400).toString() + " {{ lang._('Days') }}";
} else if (recordedtime > 3600) {
detail_text = Math.floor(recordedtime / 3600).toString() + " {{ lang._('Hours') }}";
if (set_stepsize >= 31536000) {
detail_text = Math.floor(set_stepsize / 31536000).toString() + " {{ lang._('Year(s)') }}";
} else if (set_stepsize >= 259200) {
detail_text = Math.floor(set_stepsize / 86400).toString() + " {{ lang._('Days') }}";
} else if (set_stepsize > 3600) {
detail_text = Math.floor(set_stepsize / 3600).toString() + " {{ lang._('Hours') }}";
} else {
detail_text = Math.floor(recordedtime / 60).toString() + " {{ lang._('Minutes') }}";
detail_text = Math.floor(set_stepsize / 60).toString() + " {{ lang._('Minute(s)') }}";
}
if (setcount == 0) {
zoom_buttons += '<label class="btn btn-default active"> <input type="radio" id="d' + setcount.toString() + '" name="detail" checked="checked" value="' + setcount.toString() + '" /> ' + detail_text + ' </label>';
$('#zoom').empty();
$('#zoom').append('<label class="btn btn-default active"> <input type="radio" id="d' + setcount.toString() + '" name="detail" checked="checked" value="' + setcount.toString() + '" /> ' + detail_text + ' </label>');
} else {
zoom_buttons += '<label class="btn btn-default"> <input type="radio" id="d' + setcount.toString() + '" name="detail" value="' + setcount.toString() + '" /> ' + detail_text + ' </label>';
$('#zoom').append('<label class="btn btn-default"> <input type="radio" id="d' + setcount.toString() + '" name="detail" value="' + setcount.toString() + '" /> ' + detail_text + ' </label>');
}
}
if (zoom_buttons === "") {
zoom_buttons = "<b>No data available</b>";
}
// insert zoom buttons html code
$('#zoom').html(zoom_buttons);
}
$('#stepsize').text(stepsize + " " + scale);
// Check for enabled or disabled stream, to make sure that same set stays selected after update
for (let index = 0; index < disabled.length; ++index) {
window.resize = null;
data["d3"]["data"][index]["disabled"] = disabled[index]; // disable stream if it was disabled before updating dataset
data["set"]["data"][index]["disabled"] = disabled[index]; // disable stream if it was disabled before updating dataset
}
// Create tables (general and detail)
if ($('input:radio[name=show_table]:checked').val() == 1) { // check if toggle table is on
// Setup variables for table data
let table_head; // used for table headings in html format
let table_row_data = {}; // holds row data for table
let table_view_rows = ""; // holds row data in html format
let keyname = ""; // used for name of key
let rowcounter = 0;// general row counter
let min; // holds calculated minimum value
let max; // holds calculated maximum value
let average; // holds calculated average
let t; // general date/time variable
let item; // used for name of key
let counter = 1; // used for row count
table_head = "<th>#</th>";
if ($('input:radio[name=toggle_time]:checked').val() == 1) {
table_head += "<th>{{ lang._('full date & time') }}</th>";
@ -341,56 +285,41 @@
table_head += "<th>{{ lang._('timestamp') }}</th>";
}
// Setup variables for table data
var table_head; // used for table headings in html format
var table_row_data = {}; // holds row data for table
var table_view_rows = ""; // holds row data in html format
var keyname = ""; // used for name of key
var rowcounter = 0;// general row counter
var min; // holds calculated minimum value
var max; // holds calculated maximum value
var average; // holds calculated average
var t; // general date/time variable
var item; // used for name of key
var counter = 1; // used for row count
for (let index = 0; index < data["d3"]["data"].length; ++index) {
for (let index = 0; index < data["set"]["data"].length; ++index) {
rowcounter = 0;
min = 0;
max = 0;
average = 0;
if (data["d3"]["data"][index]["disabled"] != true) {
table_head += '<th>' + data["d3"]["data"][index]["key"] + '</th>';
keyname = data["d3"]["data"][index]["key"].toString();
for (var value_index = 0; value_index < data["d3"]["data"][index]["values"].length; ++value_index) {
if (data["set"]["data"][index]["disabled"] != true) {
table_head += '<th>' + data["set"]["data"][index]["key"] + '</th>';
keyname = data["set"]["data"][index]["key"].toString();
for (let value_index = 0; value_index < data["set"]["data"][index]["values"].length; ++value_index) {
if (data["d3"]["data"][index]["values"][value_index][0] >= (from * 1000) && data["d3"]["data"][index]["values"][value_index][0] <= (to * 1000) || ( from == 0 && to == 0 )) {
if (data["set"]["data"][index]["values"][value_index][0] >= (from * 1000) && data["set"]["data"][index]["values"][value_index][0] <= (to * 1000) || ( from == 0 && to == 0 )) {
if (table_row_data[data["d3"]["data"][index]["values"][value_index][0]] === undefined) {
table_row_data[data["d3"]["data"][index]["values"][value_index][0]] = {};
if (table_row_data[data["set"]["data"][index]["values"][value_index][0]] === undefined) {
table_row_data[data["set"]["data"][index]["values"][value_index][0]] = {};
}
if (table_row_data[data["d3"]["data"][index]["values"][value_index][0]][data["d3"]["data"][index]["key"]] === undefined) {
table_row_data[data["d3"]["data"][index]["values"][value_index][0]][data["d3"]["data"][index]["key"]] = data["d3"]["data"][index]["values"][value_index][1];
if (table_row_data[data["set"]["data"][index]["values"][value_index][0]][data["set"]["data"][index]["key"]] === undefined) {
table_row_data[data["set"]["data"][index]["values"][value_index][0]][data["set"]["data"][index]["key"]] = data["set"]["data"][index]["values"][value_index][1];
}
if (csvData[rowcounter] === undefined) {
csvData[rowcounter] = {};
}
if (csvData[rowcounter]["timestamp"] === undefined) {
t = new Date(parseInt(data["d3"]["data"][index]["values"][value_index][0]));
csvData[rowcounter]["timestamp"] = data["d3"]["data"][index]["values"][value_index][0] / 1000;
t = new Date(parseInt(data["set"]["data"][index]["values"][value_index][0]));
csvData[rowcounter]["timestamp"] = data["set"]["data"][index]["values"][value_index][0] / 1000;
csvData[rowcounter]["date_time"] = t.toString();
}
csvData[rowcounter][keyname] = data["d3"]["data"][index]["values"][value_index][1];
if (data["d3"]["data"][index]["values"][value_index][1] < min) {
min = data["d3"]["data"][index]["values"][value_index][1];
csvData[rowcounter][keyname] = data["set"]["data"][index]["values"][value_index][1];
if (data["set"]["data"][index]["values"][value_index][1] < min) {
min = data["set"]["data"][index]["values"][value_index][1];
}
if (data["d3"]["data"][index]["values"][value_index][1] > max) {
max = data["d3"]["data"][index]["values"][value_index][1];
if (data["set"]["data"][index]["values"][value_index][1] > max) {
max = data["set"]["data"][index]["values"][value_index][1];
}
average += data["d3"]["data"][index]["values"][value_index][1];
average += data["set"]["data"][index]["values"][value_index][1];
++rowcounter;
}
}
@ -404,7 +333,6 @@
}
}
for ( item in min_max_average) {
table_view_rows += "<tr>";
table_view_rows += "<td>" + item + "</td>";
@ -424,7 +352,7 @@
} else {
table_view_rows += "<tr><td>" + counter.toString() + "</td><td>" + parseInt(item / 1000).toString() + "</td>";
}
for (var value in table_row_data[item]) {
for (let value in table_row_data[item]) {
table_view_rows += "<td>" + table_row_data[item][value] + "</td>";
}
++counter;
@ -448,7 +376,7 @@
chart.interactive(true);
d3.select('#chart svg')
.datum(data["d3"]["data"])
.datum(data["set"]["data"])
.transition().duration(0)
.call(chart);
@ -457,7 +385,6 @@
window.onresize = null; // clear any pending resize events
fetching_data = false;
$('#loading').hide(); // Data has been found and chart will be drawn
$('#averages').show();
$('#chart_title').show();
@ -470,14 +397,16 @@
}
} else {
alert("Error while fetching data : "+status);
$('#loading').hide();
$('#chart_title').show();
$('#chart_title').text("{{ lang._('Unable to load data') }}");
}
});
}
// convert a data Array to CSV format
function convertToCSV(args) {
var result, ctr, keys, columnDelimiter, lineDelimiter, data;
let result, ctr, keys, columnDelimiter, lineDelimiter, data;
data = args.data || null;
if (data == null || !data.length) {
@ -509,8 +438,8 @@
// download CVS file
function downloadCSV(args) {
var data, filename, link;
var csv = convertToCSV({
let data, filename, link;
let csv = convertToCSV({
data: csvData
});
if (csv == null) return;
@ -567,59 +496,36 @@
<div class="row">
<div class="col-md-12"></div>
<div class="col-md-4">
<b>{{ lang._('Zoom level') }}:</b>
<form onChange="UpdateOptions()">
<div class="btn-group btn-group-xs" data-toggle="buttons" id="zoom">
<!-- The zoom buttons are generated based upon the current dataset -->
</div>
</form>
<b>{{ lang._('Granularity') }}:</b>
<div class="btn-group btn-group-xs update_options" data-toggle="buttons" id="zoom">
<!-- The zoom buttons are generated based upon the current dataset -->
</div>
</div>
<div class="col-md-2">
<b>{{ lang._('Inverse') }}:</b>
<form onChange="UpdateOptions()">
<div class="btn-group btn-group-xs" data-toggle="buttons">
<label class="btn btn-default active">
<input type="radio" id="in0" name="inverse" checked="checked" value="0"/> {{
lang._('Off') }}
</label>
<label class="btn btn-default">
<input type="radio" id="in1" name="inverse" value="1"/> {{ lang._('On') }}
</label>
</div>
</form>
<div class="btn-group btn-group-xs update_options" data-toggle="buttons">
<label class="btn btn-default active">
<input type="radio" id="in0" name="inverse" checked="checked" value="0"/>
{{lang._('Off') }}
</label>
<label class="btn btn-default">
<input type="radio" id="in1" name="inverse" value="1"/> {{ lang._('On') }}
</label>
</div>
</div>
<div class="col-md-4">
<b>{{ lang._('Resolution') }}:</b>
<form onChange="UpdateOptions()">
<div class="btn-group btn-group-xs" data-toggle="buttons">
<label class="btn btn-default active">
<input type="radio" id="res0" name="resolution" checked="checked" value="120"/>
{{ lang._('Standard') }}
</label>
<label class="btn btn-default">
<input type="radio" id="res1" name="resolution" value="240"/> {{
lang._('Medium') }}
</label>
<label class="btn btn-default">
<input type="radio" id="res2" name="resolution" value="600"/> {{ lang._('High')
}}
</label>
</div>
</form>
</div>
<div class="col-md-2">
<b>{{ lang._('Show Tables') }}:</b>
<form onChange="UpdateOptions()">
<div class="btn-group btn-group-xs" data-toggle="buttons">
<label class="btn btn-default active">
<input type="radio" id="tab0" name="show_table" checked="checked" value="0"/> {{
lang._('Off') }}
</label>
<label class="btn btn-default">
<input type="radio" id="tab1" name="show_table" value="1"/> {{ lang._('On') }}
</label>
</div>
</form>
<div class="btn-group btn-group-xs update_options" data-toggle="buttons">
<label class="btn btn-default active">
<input type="radio" id="tab0" name="show_table" checked="checked" value="0"/> {{
lang._('Off') }}
</label>
<label class="btn btn-default">
<input type="radio" id="tab1" name="show_table" value="1"/> {{ lang._('On') }}
</label>
</div>
</div>
</div>
</div>
@ -684,17 +590,15 @@
<div class="btn-toolbar" role="toolbar">
<i>{{ lang._('Toggle Timeview') }}:</i>
<form onChange="UpdateOptions();">
<div class="btn-group" data-toggle="buttons">
<label class="btn btn-xs btn-default active">
<input type="radio" id="time0" name="toggle_time" checked="checked" value="0"/> {{
lang._('Timestamp') }}
</label>
<label class="btn btn-xs btn-default">
<input type="radio" id="time1" name="toggle_time" value="1"/> {{ lang._('Full Date & Time') }}
</label>
</div>
</form>
<div class="btn-group update_options" data-toggle="buttons">
<label class="btn btn-xs btn-default active">
<input type="radio" id="time0" name="toggle_time" checked="checked" value="0"/> {{
lang._('Timestamp') }}
</label>
<label class="btn btn-xs btn-default">
<input type="radio" id="time1" name="toggle_time" value="1"/> {{ lang._('Full Date & Time') }}
</label>
</div>
<div class="btn btn-xs btn-primary inline" onclick='downloadCSV({ filename: rrd+".csv" });'>
<i class="fa fa-download"></i> {{ lang._('Download as CSV') }}
</div>

75
src/opnsense/scripts/health/fetchData.py
View File

@ -1,7 +1,7 @@
#!/usr/local/bin/python3
"""
Copyright (c) 2015-2019 Ad Schellevis <ad@opnsense.org>
Copyright (c) 2015-2023 Ad Schellevis <ad@opnsense.org>
All rights reserved.
Redistribution and use in source and binary forms, with or without
@ -31,19 +31,72 @@
"""
import sys
import glob
import tempfile
import subprocess
import os.path
import json
from xml.etree import ElementTree
rrd_reports_dir = '/var/db/rrd'
if len(sys.argv) > 1:
filename = sys.argv[1]
def get_filename(fname):
rrd_reports_dir = '/var/db/rrd'
# suffix rrd if not already in request
if filename.split('.')[-1] != 'rrd':
filename += '.rrd'
if fname.split('.')[-1] != 'rrd':
fname += '.rrd'
# scan rrd directory for requested file
for rrdFilename in glob.glob('%s/*.rrd' % rrd_reports_dir):
if os.path.basename(rrdFilename) == filename:
subprocess.run(['/usr/local/bin/rrdtool', 'dump', rrdFilename])
break
if os.path.basename(rrdFilename).lower() == fname.lower():
return rrdFilename
def element_to_dict(elem):
dict_result = {}
if not len(elem):
return elem.text
for item in elem:
if item.tag in dict_result:
if type(dict_result[item.tag]) != list:
dict_result[item.tag] = [dict_result[item.tag]]
dict_result[item.tag].append(element_to_dict(item))
else:
dict_result[item.tag] = element_to_dict(item)
return dict_result
if len(sys.argv) > 1:
result = {}
filename = get_filename(sys.argv[1])
if filename:
sp = subprocess.run(['/usr/local/bin/rrdtool', 'dump', filename], capture_output=True, text=True)
ruleData = element_to_dict(ElementTree.fromstring(sp.stdout))
if type(ruleData.get('rra')) is list:
result['sets'] = []
for ifld in ['step', 'lastupdate']:
result[ifld] = int(ruleData[ifld]) if ifld in ruleData and ruleData[ifld].isdigit() else 0
for rra_idx, rra in enumerate(ruleData['rra']):
if rra.get('cf') == 'AVERAGE':
record = {'ds': []}
ds_count = len(ruleData['ds'])
for ds in ruleData['ds']:
record['ds'].append({
'key': ds['name'].strip() if ds.get('name') else '',
'values': []
})
for ifld in ['pdp_per_row']:
record[ifld] = int(rra[ifld]) if ifld in rra and rra[ifld].isdigit() else 0
record['step_size'] = record['pdp_per_row'] * result['step']
if 'database' in rra and 'row' in rra['database']:
last_ts = int(result['lastupdate'] / record['step_size']) * record['step_size']
first_ts = last_ts - ((len(rra['database']['row'])-1) * record['step_size'])
record['recorded_time'] = last_ts - first_ts
for idx, row in enumerate(rra['database']['row']):
this_ts = first_ts + (record['step_size'] * idx)
for vidx, v in enumerate(row['v']):
if ds_count >= vidx:
record['ds'][vidx]['values'].append([
this_ts * 1000,
float(v) if v not in ['NaN', 'inf'] else 0
])
result['sets'].append(record)
print(json.dumps(result))