diff --git a/src/opnsense/scripts/filter/lib/alias/__init__.py b/src/opnsense/scripts/filter/lib/alias/__init__.py
index 515629f4d..ab754272f 100755
--- a/src/opnsense/scripts/filter/lib/alias/__init__.py
+++ b/src/opnsense/scripts/filter/lib/alias/__init__.py
@@ -60,9 +60,7 @@ class Alias(object):
'ssl_no_verify': ssl_no_verify,
'timeout': timeout,
'interface': None,
- 'proto': 'IPv4,IPv6',
- 'password': None,
- 'authtype': None,
+ 'proto': 'IPv4,IPv6'
}
self._ttl = ttl
self._name = None
@@ -72,13 +70,10 @@ class Alias(object):
for subelem in elem:
if subelem.tag == 'type':
self._type = subelem.text
- elif subelem.tag == 'proto':
- self._properties['proto'] = subelem.text
+ self._properties['type'] = subelem.text
elif subelem.tag == 'name':
self._name = subelem.text
self._properties['name'] = self._name
- elif subelem.tag == 'interface':
- self._properties['interface'] = subelem.text
elif subelem.tag == 'ttl':
tmp = subelem.text.strip()
if len(tmp.split('.')) <= 2 and tmp.replace('.', '').isdigit():
@@ -92,12 +87,8 @@ class Alias(object):
self._items = set(sorted(subelem.text.split()))
elif subelem.tag == 'url':
self._items = set(sorted(subelem.text.split()))
- elif subelem.tag == 'authtype':
- self._properties['authtype'] = subelem.text
- elif subelem.tag == 'password':
- self._properties['password'] = subelem.text
- elif subelem.tag == 'username':
- self._properties['username'] = subelem.text
+ else:
+ self._properties[subelem.tag] = subelem.text
# we'll save the calculated hash for the unparsed alias content
self._filename_alias_hash = '/var/db/aliastables/%s.md5.txt' % self._name
@@ -117,8 +108,9 @@ class Alias(object):
:return: md5 (string)
"""
tmp = ','.join(sorted(list(self._items)))
- if self._properties['proto']:
- tmp = '%s[%s]' % (tmp, self._properties['proto'])
+ for fieldname in ['proto', 'path_expression', 'authtype', 'username', 'password']:
+ if fieldname in self._properties:
+ tmp = '%s[%s]' % (tmp, self._properties[fieldname])
return md5(tmp.encode()).hexdigest()
def changed(self):
@@ -231,7 +223,7 @@ class Alias(object):
"""
if self._type in ['host', 'network', 'networkgroup']:
return BaseContentParser(**self._properties)
- elif self._type in ['url', 'urltable']:
+ elif self._type in ['url', 'urltable', 'urljson']:
return UriParser(**self._properties)
elif self._type == 'geoip':
return GEOIP(**self._properties)
diff --git a/src/opnsense/scripts/filter/lib/alias/uri.py b/src/opnsense/scripts/filter/lib/alias/uri.py
index 229b0d57c..021184fc9 100755
--- a/src/opnsense/scripts/filter/lib/alias/uri.py
+++ b/src/opnsense/scripts/filter/lib/alias/uri.py
@@ -28,6 +28,7 @@ import re
import syslog
import requests
import urllib3
+import ujson
from .base import BaseContentParser
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@@ -41,17 +42,45 @@ class UriParser(BaseContentParser):
self._authtype = authtype
self._username = username
self._password = password
+ self._type = kwargs.get('type', None)
+ # optional path expresion
+ if kwargs.get('path_expression', None):
+ self._path_expression = kwargs['path_expression'].split('.')
+ else:
+ self._path_expression = []
+
+ def _parse_line(self, line):
+ """ return unparsed (raw) alias entries without dependencies
+ :param line: string item to parse
+ :return: iterator
+ """
+ if self._type == 'urljson':
+ try:
+ record = ujson.loads(line)
+ except ValueError:
+ record = {}
+ for field in self._path_expression:
+ if type(record) is dict and field in record:
+ record = record[field]
+ else:
+ return
+ if type(record) is str:
+ yield record
+ elif type(record) is list:
+ for item in record:
+ yield item
+ else:
+ raw_address = re.split(r'[\s,;|#]+', line)[0]
+ if raw_address and not raw_address.startswith('//'):
+ yield raw_address
def iter_addresses(self, url):
- """ return unparsed (raw) alias entries without dependencies
+ """ parse addresses, yield only valid addresses and networks
:param url: url
:return: iterator
"""
# set request parameters
- req_opts = dict()
- req_opts['url'] = url
- req_opts['stream'] = True
- req_opts['timeout'] = self._timeout
+ req_opts = {'url': url, 'stream': True, 'timeout': self._timeout}
if self._ssl_no_verify:
req_opts['verify'] = False
@@ -70,8 +99,7 @@ class UriParser(BaseContentParser):
lines = req.raw.read().decode().splitlines()
syslog.syslog(syslog.LOG_NOTICE, 'fetch alias url %s (lines: %s)' % (url, len(lines)))
for line in lines:
- raw_address = re.split(r'[\s,;|#]+', line)[0]
- if raw_address and not raw_address.startswith('//'):
+ for raw_address in self._parse_line(line):
for address in super().iter_addresses(raw_address):
yield address
else:
diff --git a/src/opnsense/service/templates/OPNsense/Filter/filter_tables.conf b/src/opnsense/service/templates/OPNsense/Filter/filter_tables.conf
index 085c42f43..9cc890cd2 100644
--- a/src/opnsense/service/templates/OPNsense/Filter/filter_tables.conf
+++ b/src/opnsense/service/templates/OPNsense/Filter/filter_tables.conf
@@ -15,6 +15,9 @@
{{ alias.type }}
{% if alias.enabled|default('0') == '0'%}
+{% elif alias.type.startswith('urljson') %}
+
{{ alias.content|e|encode_idna }}
+
{{ alias.path_expression|e }}
{% elif alias.type.startswith('urltable') %}
{{ alias.content|e|encode_idna }}
{% elif alias.type.startswith('url') %}