From cbd3beeb7dac85c50c86774c9d71c198563c5133 Mon Sep 17 00:00:00 2001
From: Ad Schellevis <ad@opnsense.org>
Date: Thu, 28 Nov 2019 16:13:28 +0100
Subject: [PATCH] log: complete new log module for
 https://github.com/opnsense/core/issues/3831 - add module (core is standard
 log directory) - add clear endpoint
 (/api/diagnostics/log/{module}/{file}/clear - parse various date formats into
 iso dates for the frontend

---
 .../Diagnostics/Api/LogController.php         | 53 ++++++++------
 .../OPNsense/Diagnostics/LogController.php    |  7 +-
 .../app/views/OPNsense/Diagnostics/log.volt   | 43 +++++++++++-
 src/opnsense/scripts/systemhealth/clearlog    | 70 +++++++++++++++++++
 src/opnsense/scripts/systemhealth/queryLog.py | 61 +++++++++++++---
 .../conf/actions.d/actions_system.conf        |  8 ++-
 6 files changed, 207 insertions(+), 35 deletions(-)
 create mode 100755 src/opnsense/scripts/systemhealth/clearlog

diff --git a/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/LogController.php b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/LogController.php
index e29b2cd64..1bd7e364d 100644
--- a/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/LogController.php
+++ b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/LogController.php
@@ -40,31 +40,40 @@ class LogController extends ApiControllerBase
 {
     public function __call($name, $arguments)
     {
+        $module = substr($name, 0, strlen($name)-6);
+        $scope = count($arguments) > 0 ? $arguments[0] : "";
+        $action = count($arguments) > 1 ? $arguments[1] : "";
         if ($this->request->isPost() && substr($name, -6) == 'Action') {
             $this->sessionClose();
-            // create filter to sanitize input data
-            $filter = new Filter();
-            $filter->add('query', new QueryFilter());
-
-            // fetch query parameters (limit results to prevent out of memory issues)
-            $itemsPerPage = $this->request->getPost('rowCount', 'int', 9999);
-            $currentPage = $this->request->getPost('current', 'int', 1);
-
-            if ($this->request->getPost('searchPhrase', 'string', '') != "") {
-                $searchPhrase = $filter->sanitize($this->request->getPost('searchPhrase'), "query");
-            } else {
-                $searchPhrase = '';
-            }
-
             $backend = new Backend();
-            $response = $backend->configdpRun("system diag log", array($itemsPerPage,
-                ($currentPage-1)*$itemsPerPage, $searchPhrase, substr($name,0, strlen($name)-6)));
-            $result = json_decode($response, true);
-            if ($result != null) {
-                $result['rowCount'] = count($result['rows']);
-                $result['total'] = $result['total_rows'];
-                $result['current'] = (int)$currentPage;
-                return $result;
+            if ($action == "clear") {
+                $backend->configdpRun("system clear log", array($module, $scope));
+                return ["status" => "ok"];
+            } else {
+                // create filter to sanitize input data
+                $filter = new Filter();
+                $filter->add('query', new QueryFilter());
+
+                // fetch query parameters (limit results to prevent out of memory issues)
+                $itemsPerPage = $this->request->getPost('rowCount', 'int', 9999);
+                $currentPage = $this->request->getPost('current', 'int', 1);
+
+                if ($this->request->getPost('searchPhrase', 'string', '') != "") {
+                    $searchPhrase = $filter->sanitize($this->request->getPost('searchPhrase'), "query");
+                } else {
+                    $searchPhrase = '';
+                }
+
+                $response = $backend->configdpRun("system diag log", array($itemsPerPage,
+                    ($currentPage-1)*$itemsPerPage, $searchPhrase, $module, $scope)
+                );
+                $result = json_decode($response, true);
+                if ($result != null) {
+                    $result['rowCount'] = count($result['rows']);
+                    $result['total'] = $result['total_rows'];
+                    $result['current'] = (int)$currentPage;
+                    return $result;
+                }
             }
         }
         return array();
diff --git a/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/LogController.php b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/LogController.php
index be4167dd5..b1a277601 100644
--- a/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/LogController.php
+++ b/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/LogController.php
@@ -37,16 +37,19 @@ use OPNsense\Base\IndexController;
  */
 class LogController extends IndexController
 {
-    public function indexAction($scope)
+    public function renderPage($module, $scope)
     {
         $this->view->pick('OPNsense/Diagnostics/log');
+        $this->view->module = $module;
         $this->view->scope = $scope;
     }
 
     public function __call($name, $arguments)
     {
         if (substr($name, -6) == 'Action') {
-            return $this->indexAction(substr($name,0, strlen($name)-6));
+            $scope = count($arguments) > 0 ? $arguments[0] : "core";
+            $module = substr($name,0, strlen($name)-6);
+            return $this->renderPage($module, $scope);
         }
     }
 }
diff --git a/src/opnsense/mvc/app/views/OPNsense/Diagnostics/log.volt b/src/opnsense/mvc/app/views/OPNsense/Diagnostics/log.volt
index ce7161ae8..650798b03 100644
--- a/src/opnsense/mvc/app/views/OPNsense/Diagnostics/log.volt
+++ b/src/opnsense/mvc/app/views/OPNsense/Diagnostics/log.volt
@@ -29,7 +29,36 @@ POSSIBILITY OF SUCH DAMAGE.
 
 <script>
     $( document ).ready(function() {
-      $("#grid-log").UIBootgrid({search:'/api/diagnostics/log/{{scope}}'});
+      $("#grid-log").UIBootgrid({
+          options:{
+              sorting:false,
+              rowSelect: false,
+              selection: false
+          },
+          search:'/api/diagnostics/log/{{module}}/{{scope}}'
+      });
+
+      $("#flushlog").on('click', function(event){
+        event.preventDefault();
+        BootstrapDialog.show({
+          type: BootstrapDialog.TYPE_DANGER,
+          title: "{{ lang._('Log') }}",
+          message: "{{ lang._('Do you really want to flush this log?') }}",
+          buttons: [{
+            label: "{{ lang._('No') }}",
+            action: function(dialogRef) {
+              dialogRef.close();
+            }}, {
+              label: "{{ lang._('Yes') }}",
+              action: function(dialogRef) {
+                  ajaxCall("/api/diagnostics/log/{{module}}/{{scope}}/clear", {}, function(){
+                      dialogRef.close();
+                      $('#grid-log').bootgrid('reload');
+                  });
+              }
+            }]
+        });
+      });
     });
 </script>
 
@@ -41,6 +70,7 @@ POSSIBILITY OF SUCH DAMAGE.
                     <thead>
                     <tr>
                         <th data-column-id="pos" data-type="numeric" data-identifier="true"  data-visible="false">#</th>
+                        <th data-column-id="timestamp" data-type="string">{{ lang._('Date') }}</th>
                         <th data-column-id="line" data-type="string">{{ lang._('Line') }}</th>
                     </tr>
                     </thead>
@@ -49,6 +79,17 @@ POSSIBILITY OF SUCH DAMAGE.
                     <tfoot>
                     </tfoot>
                 </table>
+                <table class="table">
+                    <tbody>
+                        <tr>
+                            <td>
+                              <button class="btn btn-primary pull-right" id="flushlog">
+                                  {{ lang._('Clear log') }}
+                              </button>
+                            </td>
+                        </tr>
+                    </tbody>
+                </table>
             </div>
         </div>
     </div>
diff --git a/src/opnsense/scripts/systemhealth/clearlog b/src/opnsense/scripts/systemhealth/clearlog
new file mode 100755
index 000000000..2c41f667e
--- /dev/null
+++ b/src/opnsense/scripts/systemhealth/clearlog
@@ -0,0 +1,70 @@
+#!/usr/local/bin/php
+<?php
+
+/*
+ * Copyright (C) 2019 Deciso B.V.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+ * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+require_once("config.inc");
+require_once("system.inc");
+require_once("util.inc");
+require_once("interfaces.inc");
+require_once("plugins.inc");
+
+$opts = getopt('hm:f:', array(), $optind);
+
+if (isset($opts['h']) || empty($opts['f']) || empty($opts['m'])) {
+    echo "Usage: clearlog [-h] [-m] [-f]\n\n";
+    echo "\t-h show this help text and exit\n";
+    echo "\t-m module name\n";
+    echo "\t-f filename\n";
+    exit(0);
+}
+if (isset($opts['m']) && isset($opts['f'])) {
+    $mname = basename($opts['m']);
+    $fname = basename($opts['f']);
+    if ($mname == 'core') {
+        $filename = "/var/log/{$fname}.log";
+    } else {
+        $filename = "/var/log/{$mname}/{$fname}.log";
+    }
+    if (is_file($filename)) {
+        $size = filesize($filename);
+        $handle = fopen($filename, "r");
+        fseek($handle, $size-20);
+        $is_clog = fread($handle,4) == 'CLOG';
+        fclose($handle);
+        if ($is_clog) {
+            system_clear_clog($filename);
+        } else {
+            system_clear_log($filename);
+        }
+        // XXX: should probably add some plugin hook for this.
+        if ($fname == 'dhcpd' && $mname == 'core') {
+            killbyname('dhcpd');
+            plugins_configure('dhcp');
+        }
+    }
+}
diff --git a/src/opnsense/scripts/systemhealth/queryLog.py b/src/opnsense/scripts/systemhealth/queryLog.py
index 0881ccc0c..5e16b76e6 100755
--- a/src/opnsense/scripts/systemhealth/queryLog.py
+++ b/src/opnsense/scripts/systemhealth/queryLog.py
@@ -35,22 +35,37 @@ import os.path
 import re
 import sre_constants
 import ujson
+import datetime
 sys.path.insert(0, "/usr/local/opnsense/site-python")
 from log_helper import reverse_log_reader, fetch_clog
-from params import update_params
+import argparse
+squid_ext_timeformat = r'.*(\[\d{1,2}/[A-Za-z]{3}/\d{4}:\d{1,2}:\d{1,2}:\d{1,2} \+\d{4}\]).*'
 
 if __name__ == '__main__':
     # handle parameters
-    parameters = {'limit': '0', 'offset': '0', 'filter': '', 'filename': ''}
-    update_params(parameters)
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--output', help='output type [json/text]', default='json')
+    parser.add_argument('--filter', help='filter results', default='')
+    parser.add_argument('--limit', help='limit number of results', default='')
+    parser.add_argument('--offset', help='begin at row number', default='')
+    parser.add_argument('--filename', help='log file name (excluding .log extension)', default='')
+    parser.add_argument('--module', help='module', default='core')
+    inputargs = parser.parse_args()
 
-    result = {'filters': filter, 'rows': [], 'total_rows': 0, 'origin': parameters['filename'].split('/')[-1]}
-    if parameters['filename'] != "":
-        log_filename = "/var/log/%s.log"  % os.path.basename(parameters['filename'])
-        limit = int(parameters['limit']) if parameters['limit'].isdigit()  else 0
-        offset = int(parameters['offset']) if parameters['offset'].isdigit() else 0
+    result = {'filters': filter, 'rows': [], 'total_rows': 0, 'origin': os.path.basename(inputargs.filename)}
+    if inputargs.filename != "":
+        startup_timestamp = datetime.datetime.now()
+        if inputargs.module == 'core':
+            log_filename = "/var/log/%s.log"  % os.path.basename(inputargs.filename)
+        else:
+            log_filename = "/var/log/%s/%s.log"  % (
+                os.path.basename(inputargs.module), os.path.basename(inputargs.filename)
+            )
+
+        limit = int(inputargs.limit) if inputargs.limit.isdigit()  else 0
+        offset = int(inputargs.offset) if inputargs.offset.isdigit() else 0
         try:
-            filter = parameters['filter'].replace('*', '.*').lower()
+            filter = inputargs.filter.replace('*', '.*').lower()
             if filter.find('*') == -1:
                 # no wildcard operator, assume partial match
                 filter =  ".*%s.*" % filter
@@ -68,6 +83,34 @@ if __name__ == '__main__':
                 if record['line'] != "" and filter_regexp.match(('%s' % record['line']).lower()):
                     result['total_rows'] += 1
                     if (len(result['rows']) < limit or limit == 0) and result['total_rows'] >= offset:
+                        record['timestamp'] = None
+                        if len(record['line']) > 15 and \
+                                re.match(r'(?:[01]\d|2[0123]):(?:[012345]\d):(?:[012345]\d)', record['line'][7:15]):
+                            # syslog format, strip timestamp and return actual log data
+                            ts = datetime.datetime.strptime(record['line'][0:15], "%b %d %H:%M:%S")
+                            ts = ts.replace(year=startup_timestamp.year)
+                            if (startup_timestamp - ts).days < 0:
+                                # likely previous year, (month for this year not reached yet)
+                                ts = ts.replace(year=ts.year - 1)
+                            record['timestamp'] = ts.isoformat()
+                            # strip timestamp from log line
+                            record['line'] = record['line'][16:]
+                            # strip hostname from log line
+                            record['line'] = record['line'][record['line'].find(' ')+1:].strip()
+                        elif len(record['line']) > 15 and record['line'][0:10].isdigit() and \
+                                record['line'][10] == '.' and record['line'][11:13].isdigit():
+                            # looks like an epoch
+                            ts = datetime.datetime.fromtimestamp(float(record['line'][0:13]))
+                            record['timestamp'] = ts.isoformat()
+                            # strip timestamp
+                            record['line'] = record['line'][14:].strip()
+                        elif re.match(squid_ext_timeformat, record['line']):
+                            tmp = re.match(squid_ext_timeformat, record['line'])
+                            grp = tmp.group(1)
+                            ts = datetime.datetime.strptime(grp[1:].split()[0], "%d/%b/%Y:%H:%M:%S")
+                            record['timestamp'] = ts.isoformat()
+                            # strip timestamp
+                            record['line'] = record['line'].replace(grp, '')
                         result['rows'].append(record)
                     elif result['total_rows'] > offset + limit:
                         # do not fetch data until end of file...
diff --git a/src/opnsense/service/conf/actions.d/actions_system.conf b/src/opnsense/service/conf/actions.d/actions_system.conf
index 8b34461f6..d059d6396 100644
--- a/src/opnsense/service/conf/actions.d/actions_system.conf
+++ b/src/opnsense/service/conf/actions.d/actions_system.conf
@@ -6,10 +6,16 @@ message:Show system activity
 
 [diag.log]
 command:/usr/local/opnsense/scripts/systemhealth/queryLog.py
-parameters:/limit %s /offset %s /filter %s /filename %s
+parameters:--limit %s --offset %s --filter %s  --module %s --filename %s
 type:script_output
 message:Show log
 
+[clear.log]
+command:/usr/local/opnsense/scripts/systemhealth/clearlog
+parameters:-m%s -f%s
+type:script
+message:clear log (%s %s)
+
 [list.interrupts]
 command:/usr/local/opnsense/scripts/system/list_interrupts.py
 parameters:%s