From b7662a109e35600d3bf38d77952121cf13f83eee Mon Sep 17 00:00:00 2001
From: Ad Schellevis <ad@opnsense.org>
Date: Tue, 26 Nov 2019 22:01:39 +0100
Subject: [PATCH] Logging, functional concept for
 https://github.com/opnsense/core/issues/3831 (work in progress, not finished)

---
 .../Diagnostics/Api/LogController.php         | 72 +++++++++++++++++
 .../OPNsense/Diagnostics/LogController.php    | 52 +++++++++++++
 .../app/views/OPNsense/Diagnostics/log.volt   | 55 +++++++++++++
 src/opnsense/scripts/systemhealth/queryLog.py | 77 +++++++++++++++++++
 .../conf/actions.d/actions_system.conf        |  6 ++
 5 files changed, 262 insertions(+)
 create mode 100644 src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/LogController.php
 create mode 100644 src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/LogController.php
 create mode 100644 src/opnsense/mvc/app/views/OPNsense/Diagnostics/log.volt
 create mode 100755 src/opnsense/scripts/systemhealth/queryLog.py

diff --git a/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/LogController.php b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/LogController.php
new file mode 100644
index 000000000..e29b2cd64
--- /dev/null
+++ b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/LogController.php
@@ -0,0 +1,72 @@
+<?php
+
+/*
+ * Copyright (C) 2019 Deciso B.V.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+ * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+namespace OPNsense\Diagnostics\Api;
+
+use \OPNsense\Base\ApiControllerBase;
+use \OPNsense\Core\Backend;
+use \OPNsense\Base\Filters\QueryFilter;
+use \Phalcon\Filter;
+
+/**
+ * @inherit
+ */
+class LogController extends ApiControllerBase
+{
+    public function __call($name, $arguments)
+    {
+        if ($this->request->isPost() && substr($name, -6) == 'Action') {
+            $this->sessionClose();
+            // create filter to sanitize input data
+            $filter = new Filter();
+            $filter->add('query', new QueryFilter());
+
+            // fetch query parameters (limit results to prevent out of memory issues)
+            $itemsPerPage = $this->request->getPost('rowCount', 'int', 9999);
+            $currentPage = $this->request->getPost('current', 'int', 1);
+
+            if ($this->request->getPost('searchPhrase', 'string', '') != "") {
+                $searchPhrase = $filter->sanitize($this->request->getPost('searchPhrase'), "query");
+            } else {
+                $searchPhrase = '';
+            }
+
+            $backend = new Backend();
+            $response = $backend->configdpRun("system diag log", array($itemsPerPage,
+                ($currentPage-1)*$itemsPerPage, $searchPhrase, substr($name,0, strlen($name)-6)));
+            $result = json_decode($response, true);
+            if ($result != null) {
+                $result['rowCount'] = count($result['rows']);
+                $result['total'] = $result['total_rows'];
+                $result['current'] = (int)$currentPage;
+                return $result;
+            }
+        }
+        return array();
+    }
+}
diff --git a/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/LogController.php b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/LogController.php
new file mode 100644
index 000000000..be4167dd5
--- /dev/null
+++ b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/LogController.php
@@ -0,0 +1,52 @@
+<?php
+
+/**
+ *    Copyright (C) 2019 Deciso B.V.
+ *
+ *    All rights reserved.
+ *
+ *    Redistribution and use in source and binary forms, with or without
+ *    modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ *    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ *    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ *    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ *    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+ *    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *    POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+namespace OPNsense\Diagnostics;
+
+use OPNsense\Base\IndexController;
+
+/**
+ * @inherit
+ */
+class LogController extends IndexController
+{
+    public function indexAction($scope)
+    {
+        $this->view->pick('OPNsense/Diagnostics/log');
+        $this->view->scope = $scope;
+    }
+
+    public function __call($name, $arguments)
+    {
+        if (substr($name, -6) == 'Action') {
+            return $this->indexAction(substr($name,0, strlen($name)-6));
+        }
+    }
+}
diff --git a/src/opnsense/mvc/app/views/OPNsense/Diagnostics/log.volt b/src/opnsense/mvc/app/views/OPNsense/Diagnostics/log.volt
new file mode 100644
index 000000000..ce7161ae8
--- /dev/null
+++ b/src/opnsense/mvc/app/views/OPNsense/Diagnostics/log.volt
@@ -0,0 +1,55 @@
+{#
+
+OPNsense® is Copyright © 2019 by Deciso B.V.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1.  Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+2.  Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+#}
+<script src="{{ cache_safe('/ui/js/moment-with-locales.min.js') }}"></script>
+
+<script>
+    $( document ).ready(function() {
+      $("#grid-log").UIBootgrid({search:'/api/diagnostics/log/{{scope}}'});
+    });
+</script>
+
+<div class="content-box">
+    <div class="content-box-main">
+        <div class="table-responsive">
+            <div  class="col-sm-12">
+                <table id="grid-log" class="table table-condensed table-hover table-striped table-responsive">
+                    <thead>
+                    <tr>
+                        <th data-column-id="pos" data-type="numeric" data-identifier="true"  data-visible="false">#</th>
+                        <th data-column-id="line" data-type="string">{{ lang._('Line') }}</th>
+                    </tr>
+                    </thead>
+                    <tbody>
+                    </tbody>
+                    <tfoot>
+                    </tfoot>
+                </table>
+            </div>
+        </div>
+    </div>
+</div>
diff --git a/src/opnsense/scripts/systemhealth/queryLog.py b/src/opnsense/scripts/systemhealth/queryLog.py
new file mode 100755
index 000000000..0881ccc0c
--- /dev/null
+++ b/src/opnsense/scripts/systemhealth/queryLog.py
@@ -0,0 +1,77 @@
+#!/usr/local/bin/python3
+
+"""
+    Copyright (c) 2019 Ad Schellevis <ad@opnsense.org>
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+
+    --------------------------------------------------------------------------------------
+
+    query log files
+"""
+
+import sys
+import os.path
+import re
+import sre_constants
+import ujson
+sys.path.insert(0, "/usr/local/opnsense/site-python")
+from log_helper import reverse_log_reader, fetch_clog
+from params import update_params
+
+if __name__ == '__main__':
+    # handle parameters
+    parameters = {'limit': '0', 'offset': '0', 'filter': '', 'filename': ''}
+    update_params(parameters)
+
+    result = {'filters': filter, 'rows': [], 'total_rows': 0, 'origin': parameters['filename'].split('/')[-1]}
+    if parameters['filename'] != "":
+        log_filename = "/var/log/%s.log"  % os.path.basename(parameters['filename'])
+        limit = int(parameters['limit']) if parameters['limit'].isdigit()  else 0
+        offset = int(parameters['offset']) if parameters['offset'].isdigit() else 0
+        try:
+            filter = parameters['filter'].replace('*', '.*').lower()
+            if filter.find('*') == -1:
+                # no wildcard operator, assume partial match
+                filter =  ".*%s.*" % filter
+            filter_regexp = re.compile(filter)
+        except sre_constants.error:
+            # remove illegal expression
+            filter_regexp = re.compile('.*')
+
+        if os.path.exists(log_filename):
+            try:
+                filename = fetch_clog(log_filename)
+            except Exception as e:
+                filename = log_filename
+            for record in reverse_log_reader(filename):
+                if record['line'] != "" and filter_regexp.match(('%s' % record['line']).lower()):
+                    result['total_rows'] += 1
+                    if (len(result['rows']) < limit or limit == 0) and result['total_rows'] >= offset:
+                        result['rows'].append(record)
+                    elif result['total_rows'] > offset + limit:
+                        # do not fetch data until end of file...
+                        break
+
+    # output results
+    print(ujson.dumps(result))
diff --git a/src/opnsense/service/conf/actions.d/actions_system.conf b/src/opnsense/service/conf/actions.d/actions_system.conf
index cba2e852b..8b34461f6 100644
--- a/src/opnsense/service/conf/actions.d/actions_system.conf
+++ b/src/opnsense/service/conf/actions.d/actions_system.conf
@@ -4,6 +4,12 @@ parameters:%s
 type:script_output
 message:Show system activity
 
+[diag.log]
+command:/usr/local/opnsense/scripts/systemhealth/queryLog.py
+parameters:/limit %s /offset %s /filter %s /filename %s
+type:script_output
+message:Show log
+
 [list.interrupts]
 command:/usr/local/opnsense/scripts/system/list_interrupts.py
 parameters:%s