diff --git a/src/opnsense/mvc/app/models/OPNsense/Firewall/Alias.xml b/src/opnsense/mvc/app/models/OPNsense/Firewall/Alias.xml index e64ba6ec7..1a89b0447 100644 --- a/src/opnsense/mvc/app/models/OPNsense/Firewall/Alias.xml +++ b/src/opnsense/mvc/app/models/OPNsense/Firewall/Alias.xml @@ -30,6 +30,7 @@ Port(s) URL (IPs) URL Table (IPs) + URL Table in JSON format (IPs) GeoIP Network group MAC address @@ -45,6 +46,7 @@ + Y diff --git a/src/opnsense/mvc/app/views/OPNsense/Firewall/alias.volt b/src/opnsense/mvc/app/views/OPNsense/Firewall/alias.volt index 373286a5c..f1e6af983 100644 --- a/src/opnsense/mvc/app/views/OPNsense/Firewall/alias.volt +++ b/src/opnsense/mvc/app/views/OPNsense/Firewall/alias.volt @@ -329,6 +329,7 @@ $("#row_alias\\.updatefreq").hide(); $("#row_alias\\.authtype").hide(); $("#row_alias\\.interface").hide(); + $("#row_alias\\.path_expression").hide(); $("#copy-paste").hide(); switch ($(this).val()) { case 'authgroup': @@ -357,6 +358,9 @@ $("#alias\\.proto").selectpicker('hide'); $("#alias_type_default").show(); break; + case 'urljson': + $("#row_alias\\.path_expression").show(); + /* FALLTHROUGH */ case 'urltable': $("#row_alias\\.updatefreq").show(); /* FALLTHROUGH */ @@ -897,6 +901,25 @@ + + +
+ + {{lang._('Path expression')}} +
+ + + + + + + + +
diff --git a/src/opnsense/scripts/filter/lib/alias/__init__.py b/src/opnsense/scripts/filter/lib/alias/__init__.py index 515629f4d..ab754272f 100755 --- a/src/opnsense/scripts/filter/lib/alias/__init__.py +++ b/src/opnsense/scripts/filter/lib/alias/__init__.py @@ -60,9 +60,7 @@ class Alias(object): 'ssl_no_verify': ssl_no_verify, 'timeout': timeout, 'interface': None, - 'proto': 'IPv4,IPv6', - 'password': None, - 'authtype': None, + 'proto': 'IPv4,IPv6' } self._ttl = ttl self._name = None @@ -72,13 +70,10 @@ class Alias(object): for subelem in elem: if subelem.tag == 'type': self._type = subelem.text - elif subelem.tag == 'proto': - self._properties['proto'] = subelem.text + self._properties['type'] = subelem.text elif subelem.tag == 'name': self._name = subelem.text self._properties['name'] = self._name - elif subelem.tag == 'interface': - self._properties['interface'] = subelem.text elif subelem.tag == 'ttl': tmp = subelem.text.strip() if len(tmp.split('.')) <= 2 and tmp.replace('.', '').isdigit(): @@ -92,12 +87,8 @@ class Alias(object): self._items = set(sorted(subelem.text.split())) elif subelem.tag == 'url': self._items = set(sorted(subelem.text.split())) - elif subelem.tag == 'authtype': - self._properties['authtype'] = subelem.text - elif subelem.tag == 'password': - self._properties['password'] = subelem.text - elif subelem.tag == 'username': - self._properties['username'] = subelem.text + else: + self._properties[subelem.tag] = subelem.text # we'll save the calculated hash for the unparsed alias content self._filename_alias_hash = '/var/db/aliastables/%s.md5.txt' % self._name @@ -117,8 +108,9 @@ class Alias(object): :return: md5 (string) """ tmp = ','.join(sorted(list(self._items))) - if self._properties['proto']: - tmp = '%s[%s]' % (tmp, self._properties['proto']) + for fieldname in ['proto', 'path_expression', 'authtype', 'username', 'password']: + if fieldname in self._properties: + tmp = '%s[%s]' % (tmp, self._properties[fieldname]) return md5(tmp.encode()).hexdigest() def changed(self): @@ -231,7 +223,7 @@ class Alias(object): """ if self._type in ['host', 'network', 'networkgroup']: return BaseContentParser(**self._properties) - elif self._type in ['url', 'urltable']: + elif self._type in ['url', 'urltable', 'urljson']: return UriParser(**self._properties) elif self._type == 'geoip': return GEOIP(**self._properties) diff --git a/src/opnsense/scripts/filter/lib/alias/uri.py b/src/opnsense/scripts/filter/lib/alias/uri.py index 229b0d57c..021184fc9 100755 --- a/src/opnsense/scripts/filter/lib/alias/uri.py +++ b/src/opnsense/scripts/filter/lib/alias/uri.py @@ -28,6 +28,7 @@ import re import syslog import requests import urllib3 +import ujson from .base import BaseContentParser urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -41,17 +42,45 @@ class UriParser(BaseContentParser): self._authtype = authtype self._username = username self._password = password + self._type = kwargs.get('type', None) + # optional path expresion + if kwargs.get('path_expression', None): + self._path_expression = kwargs['path_expression'].split('.') + else: + self._path_expression = [] + + def _parse_line(self, line): + """ return unparsed (raw) alias entries without dependencies + :param line: string item to parse + :return: iterator + """ + if self._type == 'urljson': + try: + record = ujson.loads(line) + except ValueError: + record = {} + for field in self._path_expression: + if type(record) is dict and field in record: + record = record[field] + else: + return + if type(record) is str: + yield record + elif type(record) is list: + for item in record: + yield item + else: + raw_address = re.split(r'[\s,;|#]+', line)[0] + if raw_address and not raw_address.startswith('//'): + yield raw_address def iter_addresses(self, url): - """ return unparsed (raw) alias entries without dependencies + """ parse addresses, yield only valid addresses and networks :param url: url :return: iterator """ # set request parameters - req_opts = dict() - req_opts['url'] = url - req_opts['stream'] = True - req_opts['timeout'] = self._timeout + req_opts = {'url': url, 'stream': True, 'timeout': self._timeout} if self._ssl_no_verify: req_opts['verify'] = False @@ -70,8 +99,7 @@ class UriParser(BaseContentParser): lines = req.raw.read().decode().splitlines() syslog.syslog(syslog.LOG_NOTICE, 'fetch alias url %s (lines: %s)' % (url, len(lines))) for line in lines: - raw_address = re.split(r'[\s,;|#]+', line)[0] - if raw_address and not raw_address.startswith('//'): + for raw_address in self._parse_line(line): for address in super().iter_addresses(raw_address): yield address else: diff --git a/src/opnsense/service/templates/OPNsense/Filter/filter_tables.conf b/src/opnsense/service/templates/OPNsense/Filter/filter_tables.conf index 085c42f43..9cc890cd2 100644 --- a/src/opnsense/service/templates/OPNsense/Filter/filter_tables.conf +++ b/src/opnsense/service/templates/OPNsense/Filter/filter_tables.conf @@ -15,6 +15,9 @@ {{ alias.type }} {% if alias.enabled|default('0') == '0'%}
+{% elif alias.type.startswith('urljson') %} + {{ alias.content|e|encode_idna }} + {{ alias.path_expression|e }} {% elif alias.type.startswith('urltable') %} {{ alias.content|e|encode_idna }} {% elif alias.type.startswith('url') %}