diff --git a/src/opnsense/scripts/filter/lib/alias.py b/src/opnsense/scripts/filter/lib/alias.py
index c1c2c088e..4f3d01da4 100644
--- a/src/opnsense/scripts/filter/lib/alias.py
+++ b/src/opnsense/scripts/filter/lib/alias.py
@@ -45,6 +45,9 @@ class Alias(object):
"""
self._known_aliases = known_aliases
self._dnsResolver = dns.resolver.Resolver()
+ self._dnsResolver.timeout = 2
+ self._is_changed = None
+ self._has_expired = None
self._ttl = ttl
self._name = None
self._type = None
@@ -58,6 +61,10 @@ class Alias(object):
self._proto = subelem.text
elif subelem.tag == 'name':
self._name = subelem.text
+ elif subelem.tag == 'ttl':
+ tmp = subelem.text.strip()
+ if len(tmp.split('.')) <= 2 and tmp.replace('.', '').isdigit():
+ self._ttl = int(float(tmp))
elif subelem.tag == 'aliasurl':
self._items = set(sorted(subelem.text.split()))
elif subelem.tag == 'address' and len(self._items) == 0:
@@ -75,6 +82,7 @@ class Alias(object):
:param address: address or network
:return: boolean
"""
+ address = address.strip()
if address.find('/') > -1:
# provided address could be a network
try:
@@ -104,7 +112,7 @@ class Alias(object):
try:
for rdata in self._dnsResolver.query(address, record_type):
yield str(rdata)
- except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN):
+ except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout):
pass
def _fetch_url(self, url, ssl_no_verify=False, timeout=120):
@@ -144,12 +152,12 @@ class Alias(object):
if do_update:
syslog.syslog(syslog.LOG_ERR, 'geoip updated (files: %s lines: %s)' % geoip.download_geolite())
- # $filename = "/usr/local/share/GeoIP/alias/".$country_code."-".$alias['proto'];
- # if (is_file($filename)) {
- # $alias_content .= file_get_contents($filename);
- # }
- if False:
- yield None
+ geoip_filename = "/usr/local/share/GeoIP/alias/%s-%s" % (geoitem, self._proto)
+ if os.path.isfile(geoip_filename):
+ with open(geoip_filename) as f_in:
+ for line in f_in:
+ for address in self._parse_address(line):
+ yield address
def items(self):
""" return unparsed (raw) alias entries without dependencies
@@ -166,21 +174,28 @@ class Alias(object):
return md5.new(','.join(sorted(list(self.items())))).hexdigest()
def changed(self):
- """ is the alias is changed
+ """ is the alias changed (cached result, if changed within this objects lifetime)
:return: boolean
"""
- if os.path.isfile(self._filename_alias_hash) and os.path.isfile(self._filename_alias_content):
- return open(self._filename_alias_hash).read().strip() != self.uniqueid()
- return True
+ if self._is_changed is None:
+ if os.path.isfile(self._filename_alias_hash) and os.path.isfile(self._filename_alias_content):
+ self._is_changed = open(self._filename_alias_hash).read().strip() != self.uniqueid()
+ else:
+ self._is_changed = True
+
+ return self._is_changed
def expired(self):
""" if this alias has an expiry (ttl), has it reached the end of it's lifetime
:return: boolean
"""
- if self._ttl > 0 and os.path.isfile(self._filename_alias_hash):
- fstat = os.stat(self._filename_alias_hash)
- return time.time() - fstat.st_mtime > self._ttl
- return True
+ if self._has_expired is None:
+ if self._ttl > 0 and os.path.isfile(self._filename_alias_hash):
+ fstat = os.stat(self._filename_alias_hash)
+ self._has_expired = time.time() - fstat.st_mtime > self._ttl
+ else:
+ self._has_expired = False
+ return self._has_expired
def resolve(self, ssl_no_verify=False, timeout=120, force=False):
""" resolve (fetch) alias content, without dependencies.
@@ -196,12 +211,13 @@ class Alias(object):
for address in self._parse_address(item):
self._resolve_content.append(address)
elif self._type in ['url', 'urltable']:
- if item.find('deciso') > -1:
- for address in self._fetch_url(item):
- self._resolve_content.append(address)
+ for address in self._fetch_url(item):
+ self._resolve_content.append(address)
elif self._type == 'geoip':
for address in self._fetch_geo(item):
self._resolve_content.append(address)
+ # de-duplicate
+ self._resolve_content = list(set(self._resolve_content))
# flush new alias content (without dependencies) to disk
open(self._filename_alias_content, 'w').write('\n'.join(self._resolve_content))
# flush md5 hash to disk
diff --git a/src/opnsense/scripts/filter/update_tables.py b/src/opnsense/scripts/filter/update_tables.py
index 06ceeb4f8..8e2041464 100755
--- a/src/opnsense/scripts/filter/update_tables.py
+++ b/src/opnsense/scripts/filter/update_tables.py
@@ -28,15 +28,20 @@
--------------------------------------------------------------------------------------
update aliases
"""
+import os
import sys
import argparse
import syslog
import xml.etree.cElementTree as ET
import syslog
+import tempfile
+import subprocess
from lib.alias import Alias
-class Aliases(object):
+class AliasParser(object):
+ """ Alias Parser class, encapsulates all aliases
+ """
def __init__(self, source_tree):
self._source_tree = source_tree
self._aliases = dict()
@@ -45,7 +50,7 @@ class Aliases(object):
known_aliases_list = map(lambda x: x.text, self._source_tree.iterfind('table/name'))
self._aliases = dict()
for elem in self._source_tree.iterfind('table'):
- alias = Alias(elem, known_aliases=known_aliases_list, ttl=60)
+ alias = Alias(elem, known_aliases=known_aliases_list)
alias.resolve()
self._aliases[alias.get_name()] = alias
@@ -64,6 +69,21 @@ class Aliases(object):
self.get_alias_deps(dep, alias_deps)
return alias_deps
+ def get(self, name):
+ """ get alias by name
+ :param name: alias name
+ :return: alias (or None if not found)
+ """
+ if name in self._aliases:
+ return self._aliases[name]
+ return None
+
+ def __iter__(self):
+ """ iterate all known aliases
+ :return: iterator
+ """
+ for alias in self._aliases:
+ yield self._aliases[alias]
if __name__ == '__main__':
status = dict()
@@ -78,6 +98,46 @@ if __name__ == '__main__':
syslog.syslog(syslog.LOG_ERR, 'filter table parse error (%s) %s' % (str(e), inputargs.source_conf))
sys.exit(-1)
- aliases = Aliases(source_tree)
+ aliases = AliasParser(source_tree)
aliases.read()
- print ('result:',aliases.get_alias_deps('recursionC'))
+ for alias in aliases:
+ # fetch alias content including dependencies
+ alias_name = alias.get_name()
+ alias_content = alias.resolve()
+ alias_changed_or_expired = max(alias.changed(), alias.expired())
+ for related_alias_name in aliases.get_alias_deps(alias_name):
+ if related_alias_name != alias_name:
+ rel_alias = aliases.get(related_alias_name)
+ if rel_alias:
+ alias_changed_or_expired = max(alias_changed_or_expired, rel_alias.changed(), rel_alias.expired())
+ alias_content += rel_alias.resolve()
+ # when the alias or any of it's dependencies has changed, generate new
+ if alias_changed_or_expired:
+ alias_content_txt = '\n'.join(sorted(alias_content))
+ if not os.path.isdir('/var/db/aliastables'):
+ if not os.path.isdir('/var/db'):
+ os.mkdir('/var/db')
+ os.mkdir('/var/db/aliastables')
+ open('/var/db/aliastables/%s.txt' % alias_name, 'w').write(alias_content_txt)
+ else:
+ alias_content_txt = open('/var/db/aliastables/%s.txt' % alias_name, 'r').read()
+
+ alias_pf_content = list()
+ with tempfile.NamedTemporaryFile() as output_stream:
+ subprocess.call(['/sbin/pfctl', '-t', alias_name, '-T', 'show'],
+ stdout=output_stream, stderr=open(os.devnull, 'wb'))
+ output_stream.seek(0)
+ for line in output_stream.read().strip().split('\n'):
+ line = line.strip()
+ if line:
+ alias_pf_content.append(line)
+
+ if len(alias_content) != len(alias_pf_content) or alias_changed_or_expired:
+ if len(alias_content) == 0:
+ # flush when target is empty
+ subprocess.call(['/sbin/pfctl', '-t', alias_name, '-T', 'flush'],
+ stdout=open(os.devnull, 'wb'), stderr=open(os.devnull, 'wb'))
+ else:
+ subprocess.call(['/sbin/pfctl', '-t', alias_name, '-T', 'replace', '-f',
+ '/var/db/aliastables/%s.txt' % alias_name],
+ stdout=open(os.devnull, 'wb'), stderr=open(os.devnull, 'wb'))
diff --git a/src/opnsense/service/templates/OPNsense/Filter/filter_tables.conf b/src/opnsense/service/templates/OPNsense/Filter/filter_tables.conf
index 9e3cd4d43..9c5623e33 100644
--- a/src/opnsense/service/templates/OPNsense/Filter/filter_tables.conf
+++ b/src/opnsense/service/templates/OPNsense/Filter/filter_tables.conf
@@ -19,7 +19,9 @@
{% endif %}{% if alias.address %}
{{ alias.address|e }}
{% endif %}{% if alias.updatefreq %}
- {{ alias.updatefreq|e }}
+ {{ alias.updatefreq|float * 86400 }}
+{% elif alias.type == 'host' %}
+ {{ system.aliasesresolveinterval|default('300') }}
{% endif %}
{% endif %}