Firewall: Aliases - support jq for alias processing, closes https://github.com/opnsense/core/issues/8277

As we already supported a dot [.] terminated format, we should support both advanced queries as simple ones using "container1.container2", by prefixing the simple format with a dot, we can offer both options using the same parser.

While comparing jq with jsonpath, the first option seems to be most practical and easier to explain.

Makefile

@@ -177,6 +177,7 @@ CORE_DEPENDS?=		ca_root_nss \
 			pkg \
 			py${CORE_PYTHON}-Jinja2 \
 			py${CORE_PYTHON}-dnspython \
+			py${CORE_PYTHON}-jq \
 			py${CORE_PYTHON}-ldap3 \
 			py${CORE_PYTHON}-netaddr \
 			py${CORE_PYTHON}-requests \

src/opnsense/mvc/app/views/OPNsense/Firewall/alias.volt

@@ -380,6 +380,7 @@
                                 break;
                         }
                     });
+                    $("#alias\\.authtype").change();
                     /* FALLTHROUGH */
                 default:
                     $("#alias_type_default").show();
@@ -918,7 +919,7 @@
                                         <input type="text" class="form-control" size="50" id="alias.path_expression"/>
                                         <div class="hidden" data-for="help_for_alias.path_expression">
                                             <small>
-                                                {{lang._('Simplified expression to select a field inside a container, a dot [.] is used as field separator (e.g. container.fieldname).')}}
+                                                {{lang._('Simplified expression to select a field inside a container, a dot [.] is used as field separator (e.g. container.fieldname). Expressions using the jq language are also supported.')}}
                                             </small>
                                         </div>
                                     </td>

src/opnsense/scripts/filter/lib/alias/uri.py

@@ -27,8 +27,9 @@
 import re
 import syslog
 import requests
+import time
 import urllib3
-import ujson
+import jq
 from .base import BaseContentParser
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -44,35 +45,16 @@ class UriParser(BaseContentParser):
         self._password = password
         self._type = kwargs.get('type', None)
         # optional path expresion
-        if kwargs.get('path_expression', None):
+        self._path_expression = kwargs.get('path_expression', '')
-            self._path_expression = kwargs['path_expression'].split('.')
-        else:
-            self._path_expression = []
 
     def _parse_line(self, line):
         """ return unparsed (raw) alias entries without dependencies
             :param line: string item to parse
             :return: iterator
         """
-        if self._type == 'urljson':
+        raw_address = re.split(r'[\s,;|#]+', line)[0]
-            try:
+        if raw_address and not raw_address.startswith('//'):
-                record = ujson.loads(line)
+            yield raw_address
-            except ValueError:
-                record = {}
-            for field in self._path_expression:
-                if type(record) is dict and field in record:
-                    record = record[field]
-                else:
-                    return
-            if type(record) is str:
-                yield record
-            elif type(record) is list:
-                for item in record:
-                    yield item
-        else:
-            raw_address = re.split(r'[\s,;|#]+', line)[0]
-            if raw_address and not raw_address.startswith('//'):
-                yield raw_address
 
     def iter_addresses(self, url):
         """ parse addresses, yield only valid addresses and networks
@@ -96,15 +78,39 @@ class UriParser(BaseContentParser):
             if req.status_code == 200:
                 # only handle content if response is correct
                 req.raw.decode_content = True
-                lines = req.raw.read().decode().splitlines()
+                stime = time.time()
-                syslog.syslog(syslog.LOG_NOTICE, 'fetch alias url %s (lines: %s)' % (url, len(lines)))
+                if self._type == 'urljson':
-                for line in lines:
+                    data = req.raw.read().decode()
-                    for raw_address in self._parse_line(line):
+                    syslog.syslog(syslog.LOG_NOTICE, 'fetch alias url %s (bytes: %s)' % (url, len(data)))
-                        for address in super().iter_addresses(raw_address):
+                    # also support existing a.b format by prefixing [.], only raise exceptions on original input
-                            yield address
+                    jqc = None
+                    jqc_exception = None
+                    for expr in [self._path_expression, ".%s" % self._path_expression]:
+                        try:
+                            jqc = jq.compile(expr)
+                        except Exception as e:
+                            if jqc_exception is None:
+                                jqc_exception = e
+
+                    if jqc is None:
+                        raise jqc_exception
+
+                    for raw_address in iter(jqc.input_text(data)):
+                        if raw_address:
+                            for address in super().iter_addresses(raw_address):
+                                yield address
+                else:
+                    lines = req.raw.read().decode().splitlines()
+                    syslog.syslog(syslog.LOG_NOTICE, 'fetch alias url %s (lines: %s)' % (url, len(lines)))
+                    for line in lines:
+                        for raw_address in self._parse_line(line):
+                            for address in super().iter_addresses(raw_address):
+                                yield address
+
+                syslog.syslog(syslog.LOG_NOTICE, 'processing alias url %s took %0.2fs' % (url, time.time() - stime))
             else:
                 syslog.syslog(syslog.LOG_ERR, 'error fetching alias url %s [http_code:%s]' % (url, req.status_code))
                 raise IOError('error fetching alias url %s' % (url))
-        except:
+        except Exception as e:
-            syslog.syslog(syslog.LOG_ERR, 'error fetching alias url %s' % (url))
+            syslog.syslog(syslog.LOG_ERR, 'error fetching alias url %s (%s)' % (url, str(e).replace("\n", ' ')))
             raise IOError('error fetching alias url %s' % (url))