firewall, alias. work in progress https://github.com/opnsense/core/issues/1971

src/opnsense/scripts/filter/lib/alias.py

@@ -45,6 +45,9 @@ class Alias(object):
         """
         self._known_aliases = known_aliases
         self._dnsResolver = dns.resolver.Resolver()
+        self._dnsResolver.timeout = 2
+        self._is_changed = None
+        self._has_expired = None
         self._ttl = ttl
         self._name = None
         self._type = None
@@ -58,6 +61,10 @@ class Alias(object):
                 self._proto = subelem.text
             elif subelem.tag == 'name':
                 self._name = subelem.text
+            elif subelem.tag == 'ttl':
+                tmp = subelem.text.strip()
+                if len(tmp.split('.')) <= 2 and tmp.replace('.', '').isdigit():
+                    self._ttl = int(float(tmp))
             elif subelem.tag == 'aliasurl':
                 self._items = set(sorted(subelem.text.split()))
             elif subelem.tag == 'address' and len(self._items) == 0:
@@ -75,6 +82,7 @@ class Alias(object):
             :param address: address or network
             :return: boolean
         """
+        address = address.strip()
         if address.find('/') > -1:
             # provided address could be a network
             try:
@@ -104,7 +112,7 @@ class Alias(object):
             try:
                 for rdata in self._dnsResolver.query(address, record_type):
                     yield str(rdata)
-            except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN):
+            except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout):
                 pass
 
     def _fetch_url(self, url, ssl_no_verify=False, timeout=120):
@@ -144,12 +152,12 @@ class Alias(object):
         if do_update:
             syslog.syslog(syslog.LOG_ERR, 'geoip updated (files: %s lines: %s)' % geoip.download_geolite())
 
-        # $filename = "/usr/local/share/GeoIP/alias/".$country_code."-".$alias['proto'];
-        # if (is_file($filename)) {
-        #     $alias_content .= file_get_contents($filename);
-        # }
-        if False:
-            yield None
+        geoip_filename = "/usr/local/share/GeoIP/alias/%s-%s" % (geoitem, self._proto)
+        if os.path.isfile(geoip_filename):
+            with open(geoip_filename) as f_in:
+                for line in f_in:
+                    for address in self._parse_address(line):
+                        yield address
 
     def items(self):
         """ return unparsed (raw) alias entries without dependencies
@@ -166,21 +174,28 @@ class Alias(object):
         return md5.new(','.join(sorted(list(self.items())))).hexdigest()
 
     def changed(self):
-        """ is the alias is changed
+        """ is the alias changed (cached result, if changed within this objects lifetime)
             :return: boolean
         """
-        if os.path.isfile(self._filename_alias_hash) and os.path.isfile(self._filename_alias_content):
-            return open(self._filename_alias_hash).read().strip() != self.uniqueid()
-        return True
+        if self._is_changed is None:
+            if os.path.isfile(self._filename_alias_hash) and os.path.isfile(self._filename_alias_content):
+                self._is_changed = open(self._filename_alias_hash).read().strip() != self.uniqueid()
+            else:
+                self._is_changed = True
+
+        return self._is_changed
 
     def expired(self):
         """ if this alias has an expiry (ttl), has it reached the end of it's lifetime
             :return: boolean
         """
-        if self._ttl > 0 and os.path.isfile(self._filename_alias_hash):
-            fstat = os.stat(self._filename_alias_hash)
-            return time.time() - fstat.st_mtime > self._ttl
-        return True
+        if self._has_expired is None:
+            if self._ttl > 0 and os.path.isfile(self._filename_alias_hash):
+                fstat = os.stat(self._filename_alias_hash)
+                self._has_expired = time.time() - fstat.st_mtime > self._ttl
+            else:
+                self._has_expired = False
+        return self._has_expired
 
     def resolve(self, ssl_no_verify=False, timeout=120, force=False):
         """ resolve (fetch) alias content, without dependencies.
@@ -196,12 +211,13 @@ class Alias(object):
                         for address in self._parse_address(item):
                             self._resolve_content.append(address)
                     elif self._type in ['url', 'urltable']:
-                        if item.find('deciso') > -1:
-                            for address in self._fetch_url(item):
-                                self._resolve_content.append(address)
+                        for address in self._fetch_url(item):
+                            self._resolve_content.append(address)
                     elif self._type == 'geoip':
                         for address in self._fetch_geo(item):
                             self._resolve_content.append(address)
+                # de-duplicate
+                self._resolve_content = list(set(self._resolve_content))
                 # flush new alias content (without dependencies) to disk
                 open(self._filename_alias_content, 'w').write('\n'.join(self._resolve_content))
                 # flush md5 hash to disk

src/opnsense/scripts/filter/update_tables.py

@@ -28,15 +28,20 @@
     --------------------------------------------------------------------------------------
     update aliases
 """
+import os
 import sys
 import argparse
 import syslog
 import xml.etree.cElementTree as ET
 import syslog
+import tempfile
+import subprocess
 from lib.alias import Alias
 
 
-class Aliases(object):
+class AliasParser(object):
+    """ Alias Parser class, encapsulates all aliases
+    """
     def __init__(self, source_tree):
         self._source_tree = source_tree
         self._aliases = dict()
@@ -45,7 +50,7 @@ class Aliases(object):
         known_aliases_list = map(lambda x: x.text, self._source_tree.iterfind('table/name'))
         self._aliases = dict()
         for elem in self._source_tree.iterfind('table'):
-            alias = Alias(elem, known_aliases=known_aliases_list, ttl=60)
+            alias = Alias(elem, known_aliases=known_aliases_list)
             alias.resolve()
             self._aliases[alias.get_name()] = alias
 
@@ -64,6 +69,21 @@ class Aliases(object):
                     self.get_alias_deps(dep, alias_deps)
         return alias_deps
 
+    def get(self, name):
+        """ get alias by name
+            :param name: alias name
+            :return: alias (or None if not found)
+        """
+        if name in self._aliases:
+            return self._aliases[name]
+        return None
+
+    def __iter__(self):
+        """ iterate all known aliases
+            :return: iterator
+        """
+        for alias in self._aliases:
+            yield self._aliases[alias]
 
 if __name__ == '__main__':
     status = dict()
@@ -78,6 +98,46 @@ if __name__ == '__main__':
         syslog.syslog(syslog.LOG_ERR, 'filter table parse error (%s) %s' % (str(e), inputargs.source_conf))
         sys.exit(-1)
 
-    aliases = Aliases(source_tree)
+    aliases = AliasParser(source_tree)
     aliases.read()
-    print ('result:',aliases.get_alias_deps('recursionC'))
+    for alias in aliases:
+        # fetch alias content including dependencies
+        alias_name = alias.get_name()
+        alias_content = alias.resolve()
+        alias_changed_or_expired = max(alias.changed(), alias.expired())
+        for related_alias_name in aliases.get_alias_deps(alias_name):
+            if related_alias_name != alias_name:
+                rel_alias = aliases.get(related_alias_name)
+                if rel_alias:
+                    alias_changed_or_expired = max(alias_changed_or_expired, rel_alias.changed(), rel_alias.expired())
+                    alias_content += rel_alias.resolve()
+        # when the alias or any of it's dependencies has changed, generate new
+        if alias_changed_or_expired:
+            alias_content_txt = '\n'.join(sorted(alias_content))
+            if not os.path.isdir('/var/db/aliastables'):
+                if not os.path.isdir('/var/db'):
+                    os.mkdir('/var/db')
+                os.mkdir('/var/db/aliastables')
+            open('/var/db/aliastables/%s.txt' % alias_name, 'w').write(alias_content_txt)
+        else:
+            alias_content_txt = open('/var/db/aliastables/%s.txt' % alias_name, 'r').read()
+
+        alias_pf_content = list()
+        with tempfile.NamedTemporaryFile() as output_stream:
+            subprocess.call(['/sbin/pfctl', '-t', alias_name, '-T', 'show'],
+                            stdout=output_stream, stderr=open(os.devnull, 'wb'))
+            output_stream.seek(0)
+            for line in output_stream.read().strip().split('\n'):
+                line = line.strip()
+                if line:
+                    alias_pf_content.append(line)
+
+        if len(alias_content) != len(alias_pf_content) or alias_changed_or_expired:
+            if len(alias_content) == 0:
+                # flush when target is empty
+                subprocess.call(['/sbin/pfctl', '-t', alias_name, '-T', 'flush'],
+                                stdout=open(os.devnull, 'wb'), stderr=open(os.devnull, 'wb'))
+            else:
+                subprocess.call(['/sbin/pfctl', '-t', alias_name, '-T', 'replace', '-f',
+                                 '/var/db/aliastables/%s.txt' % alias_name],
+                                 stdout=open(os.devnull, 'wb'), stderr=open(os.devnull, 'wb'))

src/opnsense/service/templates/OPNsense/Filter/filter_tables.conf

@@ -19,7 +19,9 @@
 {%      endif %}{% if alias.address %}
       <address>{{ alias.address|e }}</address>
 {%      endif %}{% if alias.updatefreq %}
-      <updatefreq>{{ alias.updatefreq|e }}</updatefreq>
+      <ttl>{{ alias.updatefreq|float * 86400 }}</ttl>
+{% elif alias.type == 'host' %}
+      <ttl>{{ system.aliasesresolveinterval|default('300') }}</ttl>
 {%      endif %}
     </table>
 {%    endif %}