Netflow/insight and the endless UFS instability... in some cases sqlite can't figure out database inconsistencies by itself. In these cases it is practical to force a repair (rebuild) of the underlaying files manually.

This commit adds an option in reporting_settings.php to request a dump/import from https://github.com/opnsense/core/blob/18.1.9/src/opnsense/site-python/sqlite3_helper.py#L69

src/opnsense/scripts/netflow/flowd_aggregate.py

@@ -30,6 +30,7 @@
 import time
 import os
 import sys
+import fcntl
 import signal
 import glob
 import copy
@@ -190,6 +191,17 @@ if len(sys.argv) > 1 and 'console' in sys.argv[1:]:
         print s.getvalue()
     else:
         Main()
+elif len(sys.argv) > 1 and 'repair' in sys.argv[1:]:
+    # force a database repair, when
+    try:
+        lck = open('/var/run/flowd_aggregate.pid', 'a+')
+        fcntl.flock(lck, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        check_and_repair(filename_mask='/var/netflow/*.sqlite', force_repair=True)
+        lck.close()
+        os.remove('/var/run/flowd_aggregate.pid')
+    except IOError:
+        # already running, exit status 99
+        sys.exit(99)
 else:
     # Daemonize flowd aggregator
     daemon = Daemonize(app="flowd_aggregate", pid='/var/run/flowd_aggregate.pid', action=Main)

src/opnsense/service/conf/actions.d/actions_netflow.conf

@@ -58,6 +58,12 @@ command:/usr/local/etc/rc.d/flowd_aggregate start
 type:script
 message:start netflow data aggregator
 
+[aggregate.repair]
+command:/usr/local/opnsense/scripts/netflow/flowd_aggregate.py repair && /usr/local/etc/rc.d/flowd_aggregate start
+parameters:
+type:script
+message:force database repair
+
 [aggregate.restart]
 command:/usr/local/etc/rc.d/flowd_aggregate restart
 type:script

src/opnsense/site-python/sqlite3_helper.py

@@ -32,7 +32,7 @@ import sqlite3
 import syslog
 import os
 
-def check_and_repair(filename_mask):
+def check_and_repair(filename_mask, force_repair=False):
     """ check and repair sqlite databases
     :param filename_mask: filenames (glob pattern)
     :return: None
@@ -54,10 +54,12 @@ def check_and_repair(filename_mask):
         # force a repair when corrupted, using a dump / import
         if cur is not None:
             try:
+                if force_repair:
+                    raise sqlite3.DatabaseError("Requested forced repair")
                 cur.execute('pragma integrity_check')
                 cur.execute('analyze')
             except sqlite3.DatabaseError, e:
-                if e.message.find('malformed') > -1:
+                if e.message.find('malformed') > -1 or force_repair:
                     syslog.syslog(syslog.LOG_ERR, "sqlite3 repair %s" % filename)
                     filename_tmp = '%s.fix'%filename
                     filename_sql = '%s.sql'%filename

src/www/reporting_settings.php

@@ -51,6 +51,10 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') {
     } elseif (!empty($pconfig['action']) && $pconfig['action'] == "flush_netflow") {
         $savemsg = gettext('All local netflow data has been cleared.');
         configd_run("netflow flush");
+    } elseif (!empty($pconfig['action']) && $pconfig['action'] == "repair_netflow") {
+        $savemsg = gettext('Database repair in progress, daemon will start when done.');
+        configd_run("netflow aggregate stop");
+        configd_run("netflow aggregate repair", true);
     } else {
         $rrdcfg['enable'] = !empty($pconfig['rrdenable']);
         $savemsg = get_std_save_message();
@@ -116,6 +120,26 @@ $(document).ready(function() {
                 }]
         });
     });
+    $("#repair_netflow").click(function(event){
+        event.preventDefault();
+        BootstrapDialog.show({
+            type:BootstrapDialog.TYPE_DANGER,
+            title: "<?= gettext("Netflow/Insight");?>",
+            message: "<?=gettext('Do you really want to force a repair of the netflow data? This might take a while.');?>",
+            buttons: [{
+                    label: "<?= gettext("No");?>",
+                    action: function(dialogRef) {
+                        dialogRef.close();
+                    }}, {
+                      label: "<?= gettext("Yes");?>",
+                      action: function(dialogRef) {
+                        $("#action").val("repair_netflow");
+                        $("#iform").submit()
+                    }
+                }]
+        });
+    });
+
     $(".act_flush").click(function(event){
         var filename = $(this).data('id');
         event.preventDefault();
@@ -173,6 +197,7 @@ $(document).ready(function() {
                       <input name="Submit" type="submit" class="btn btn-primary" value="<?=gettext("Save");?>" />
                       <input type="button" name="ResetRRD" id="ResetRRD" class="btn btn-default" value="<?=gettext("Reset RRD Data");?>" />
                       <input type="button" id="flush_netflow" class="btn btn-default" value="<?=gettext("Reset Netflow Data");?>" />
+                      <input type="button" id="repair_netflow" class="btn btn-default" value="<?=gettext("Repair (forced)");?>" />
                     </td>
                   </tr>
                   <tr>