From e5d6acd2eb6865ad3fdb95cb3a6c2a4030d14f8c Mon Sep 17 00:00:00 2001 From: Stephan de Wit Date: Mon, 2 Jan 2023 14:06:53 +0100 Subject: [PATCH] Unbound / Blocklists: add exact domain blocking and integrate into overview page (#6205) This include the ability to whitelist it from the same page as well. Relevant to both the top passed/blocked domains, as well as the detailed query grid. blocklists.py has been modified in such a way that it will detect whether it needs to start the download process or simply administrate locally. The latter currently only happens when custom domains for blocking have been added/removed by a user. The reasoning is that we can easily extend/shrink the current blocklist when it comes to blocking exact domains as this is handled on the incoming side. However, while we can modify the current list to accomodate a new whitelist entry (which can be regex), we (currently) cannot know which domains were skipped in the process of retrieving them in the first place if a user explicitly removes a whitelist entry. Therefore we decide to re-run the download on a whitelist action. furthermore, the updateBlocklistAction in the controller administrates how the model is updated (e.g. when a blocked item is whitelisted, it should be removed from the blocklist model entry and added to the whitelist) In the future we could optimize the whole process by checking if a remote file has changed in date or size. --- .../Unbound/Api/SettingsController.php | 52 ++++++++ .../OPNsense/Unbound/forms/dnsbl.xml | 8 ++ .../app/models/OPNsense/Unbound/Unbound.xml | 3 + .../app/views/OPNsense/Unbound/overview.volt | 79 ++++++++++-- src/opnsense/scripts/unbound/blocklists.py | 118 ++++++++++++------ .../OPNsense/Unbound/core/blocklists.conf | 8 ++ 6 files changed, 221 insertions(+), 47 deletions(-) diff --git a/src/opnsense/mvc/app/controllers/OPNsense/Unbound/Api/SettingsController.php b/src/opnsense/mvc/app/controllers/OPNsense/Unbound/Api/SettingsController.php index 1b36c80b6..245e602a3 100644 --- a/src/opnsense/mvc/app/controllers/OPNsense/Unbound/Api/SettingsController.php +++ b/src/opnsense/mvc/app/controllers/OPNsense/Unbound/Api/SettingsController.php @@ -40,6 +40,58 @@ class SettingsController extends ApiMutableModelControllerBase private $type = 'dot'; + public function updateBlocklistAction() + { + $result = ["status" => "failed"]; + if ($this->request->isPost() && $this->request->hasPost('domain') && $this->request->hasPost('type')) { + Config::getInstance()->lock(); + $domain = $this->request->getPost('domain'); + $type = $this->request->getPost('type'); + $mdl = $this->getModel(); + $item = $mdl->getNodeByReference('dnsbl.'.$type); + + if ($item != null) { + $remove = function($csv, $item) { + $parts = explode(',', $csv); + while(($i = array_search($item, $parts)) !== false) { + unset($parts[$i]); + } + return implode(',', $parts); + }; + + // strip off any trailing dot + $value = rtrim($domain, '.'); + $wl = (string)$mdl->dnsbl->whitelists; + $bl = (string)$mdl->dnsbl->blocklists; + + if (strpos((string)$mdl->dnsbl->$type, $value) !== false) { + // value already in model, no need to re-run a potentially + // expensive dnsbl action + return ["status" => "OK"]; + } + + // Check if domains should be switched around in the model + if ($type == 'whitelists' && strpos($bl, $value) !== false) { + $mdl->dnsbl->blocklists = $remove((string)$mdl->dnsbl->blocklists, $value); + } elseif ($type == 'blocklists' && strpos($wl, $value) !== false) { + $mdl->dnsbl->whitelists = $remove((string)$mdl->dnsbl->whitelists, $value); + } + + // update the model + $list = array_filter(explode(',', (string)$item)); + $list[] = $value; + $mdl->dnsbl->$type = implode(',', $list); + + $mdl->serializeToConfig(); + Config::getInstance()->save(); + + $service = new \OPNsense\Unbound\Api\ServiceController(); + $result = $service->dnsblAction(); + } + } + return $result; + } + public function getNameserversAction() { if ($this->request->isGet()) { diff --git a/src/opnsense/mvc/app/controllers/OPNsense/Unbound/forms/dnsbl.xml b/src/opnsense/mvc/app/controllers/OPNsense/Unbound/forms/dnsbl.xml index a139ab82f..18acf8413 100644 --- a/src/opnsense/mvc/app/controllers/OPNsense/Unbound/forms/dnsbl.xml +++ b/src/opnsense/mvc/app/controllers/OPNsense/Unbound/forms/dnsbl.xml @@ -28,6 +28,14 @@ true List of domains to whitelist. You can use regular expressions. + + unbound.dnsbl.blocklists + + select_multiple + + true + List of domains to blocklist. Only exact matches are supported. + unbound.dnsbl.address diff --git a/src/opnsense/mvc/app/models/OPNsense/Unbound/Unbound.xml b/src/opnsense/mvc/app/models/OPNsense/Unbound/Unbound.xml index 56c2228cc..2a85e7d30 100644 --- a/src/opnsense/mvc/app/models/OPNsense/Unbound/Unbound.xml +++ b/src/opnsense/mvc/app/models/OPNsense/Unbound/Unbound.xml @@ -152,6 +152,9 @@ N + + N +
N N diff --git a/src/opnsense/mvc/app/views/OPNsense/Unbound/overview.volt b/src/opnsense/mvc/app/views/OPNsense/Unbound/overview.volt index 11c6cfe22..e5302493d 100644 --- a/src/opnsense/mvc/app/views/OPNsense/Unbound/overview.volt +++ b/src/opnsense/mvc/app/views/OPNsense/Unbound/overview.volt @@ -347,32 +347,37 @@ }); } - function createTopList(id, data) { + function createTopList(id, data, type) { let idx = 1; for (const [domain, statObj] of Object.entries(data)) { + let class_type = type == "pass" ? "block-domain" : "whitelist-domain"; + let icon_type = type == "pass" ? "fa fa-ban text-danger" : "fa fa-pencil text-info"; let bl = statObj.hasOwnProperty('blocklist') ? '(' + statObj.blocklist + ')' : ''; $('#' + id).append( - '
  • ' + + '
  • ' + idx + '. ' + domain + ' ' + bl + - ''+ statObj.total +' (' + statObj.pcnt +'%)' + - '
    • ' + ''+ statObj.total +' (' + statObj.pcnt +'%)' + + '' + + '' ) idx++; } + $(".block-domain").tooltip().attr('title', "{{ lang._('Block Domain') }}"); + $(".whitelist-domain").tooltip().attr('title', "{{ lang._('Whitelist Domain') }}"); } function create_or_update_totals() { ajaxGet('/api/unbound/overview/totals/10', {}, function(data, status) { - $('#top').not(':first').empty(); - $('#top-blocked').not(':first').empty(); + $('.top-item').remove(); $('#totalCounter').html(data.total); $('#blockedCounter').html(data.blocked.total + " (" + data.blocked.pcnt + "%)"); $('#sizeCounter').html(data.blocklist_size); $('#resolvedCounter').html(data.resolved.total + " (" + data.resolved.pcnt + "%)"); - createTopList('top', data.top); - createTopList('top-blocked', data.top_blocked); + createTopList('top', data.top, 'pass'); + createTopList('top-blocked', data.top_blocked, 'block'); $('#top li:nth-child(even)').addClass('odd-bg'); $('#top-blocked li:nth-child(even)').addClass('odd-bg'); @@ -445,6 +450,26 @@ updateClientChart(this.checked); }) + $(document).on('click', '.block-domain', function() { + $(this).remove("i").html(''); + ajaxCall('/api/unbound/settings/updateBlocklist', { + 'domain': $(this).data('value'), + 'type': 'blocklists' + }, function(data, status) { + $(".block-domain").remove("i").html(''); + }); + }); + + $(document).on('click', '.whitelist-domain', function() { + $(this).remove("i").html(''); + ajaxCall('/api/unbound/settings/updateBlocklist', { + 'domain': $(this).data('value'), + 'type': 'whitelists' + }, function(data, status) { + $(".whitelist-domain").remove("i").html(''); + }); + }); + do_startup().done(function() { $('.wrapper').show(); }).fail(function() { @@ -467,7 +492,16 @@ }, "resolveformatter": function (column, row) { return row.resolve_time_ms + 'ms'; - } + }, + "commands": function (column, row) { + let btn = ''; + if (row.action == 'Pass') { + btn = ' '; + } else if (row.action == 'Block') { + btn = ''; + } + return btn; + }, }, statusMapping: { 0: "query-success", @@ -477,6 +511,30 @@ 4: "danger" } } + }).on("loaded.rs.jquery.bootgrid", function (e) { + $(".block-domain").tooltip().attr('title', "{{ lang._('Block Domain') }}"); + $(".whitelist-domain").tooltip().attr('title', "{{ lang._('Whitelist Domain') }}"); + grid_queries.find(".block-domain").on("click", function(e) { + $(this).remove("i").html(''); + ajaxCall('/api/unbound/settings/updateBlocklist', { + 'domain': $(this).data('domain'), + 'type': 'blocklists', + }, function(data, status) { + let btn = grid_queries.find(".block-domain"); + btn.remove("i").html(''); + }); + }); + + grid_queries.find(".whitelist-domain").on("click", function(e) { + $(this).remove("i").html(''); + ajaxCall('/api/unbound/settings/updateBlocklist', { + 'domain': $(this).data('domain'), + 'type': 'whitelists', + }, function(data, status) { + let btn = grid_queries.find(".whitelist-domain"); + btn.remove("i").html(''); + }); + }); }); } if (e.target.id == 'query_overview_tab') { @@ -569,7 +627,7 @@
    - +
    @@ -658,6 +716,7 @@ {{ lang._('Resolve time') }} {{ lang._('TTL') }} {{ lang._('Blocklist') }} + {{ lang._('Command') }} diff --git a/src/opnsense/scripts/unbound/blocklists.py b/src/opnsense/scripts/unbound/blocklists.py index 3e42c58eb..05a36acb5 100755 --- a/src/opnsense/scripts/unbound/blocklists.py +++ b/src/opnsense/scripts/unbound/blocklists.py @@ -35,7 +35,7 @@ import time import fcntl from configparser import ConfigParser import requests -import json +import ujson def uri_reader(uri): req_opts = { @@ -76,8 +76,6 @@ def uri_reader(uri): 'blocklist download : unable to download file from %s (status_code: %d)' % (uri, req.status_code) ) - - if __name__ == '__main__': # check for a running download process, this may take a while so it's better to check... try: @@ -100,9 +98,33 @@ if __name__ == '__main__': 'data': {}, 'config': {} } + skip_download = False if os.path.exists('/tmp/unbound-blocklists.conf'): cnf = ConfigParser() cnf.read('/tmp/unbound-blocklists.conf') + + cnf_cache = ConfigParser() + if os.path.exists('/tmp/unbound-blocklists.conf.cache'): + cnf_cache.read('/tmp/unbound-blocklists.conf.cache') + else: + cnf_cache.read('/tmp/unbound-blocklists.conf') + + if cnf.sections() and cnf_cache.sections(): + # get the difference between the old and new configuration, there won't be any + # if we're starting up, so it will proceed as normal. + diff_cnf = {d: set(map(tuple, v.items())) for d,v in cnf._sections.items()} + diff_cnf_cache = {d: set(map(tuple, v.items())) for d,v in cnf_cache._sections.items()} + diffs_added = {header: diff_cnf[header] - diff_cnf_cache[header] for header, _ in diff_cnf.items()} + diffs_removed = {header: diff_cnf_cache[header] - diff_cnf[header] for header, _ in diff_cnf.items()} + + # we can only skip download if the include option has changed, but it must proceed + # if any other option has changed + if (diffs_added['include'] or diffs_removed['include']): + skip_download = True + for (a, r) in zip(diffs_added, diffs_removed): + if (a != 'include' and r != 'include') and (diffs_added[a] or diffs_removed[r]): + skip_download = False + # exclude (white) lists, compile to regex to be used to filter blocklist entries if cnf.has_section('exclude'): exclude_list = set() @@ -123,48 +145,70 @@ if __name__ == '__main__': whitelist_pattern = re.compile(wp, re.IGNORECASE) syslog.syslog(syslog.LOG_NOTICE, 'blocklist download : exclude domains matching %s' % wp) - # fetch all blocklists - if cnf.has_section('settings'): - if cnf.has_option('settings', 'address'): - blocklist_items['config']['dst_addr'] = cnf.get('settings', 'address') - if cnf.has_option('settings', 'rcode'): - blocklist_items['config']['rcode'] = cnf.get('settings', 'rcode') - if cnf.has_section('blocklists'): - for blocklist in cnf['blocklists']: - list_type = blocklist.split('_', 1) - bl_shortcode = 'Custom' if list_type[0] == 'custom' else list_type[1] - file_stats = {'uri': cnf['blocklists'][blocklist], 'skip' : 0, 'blocklist': 0, 'lines' :0} - for line in uri_reader(cnf['blocklists'][blocklist]): - file_stats['lines'] += 1 - # cut line into parts before comment marker (if any) - tmp = line.split('#')[0].split() - entry = None - while tmp: - entry = tmp.pop(-1) - if entry not in ['127.0.0.1', '0.0.0.0']: - break - if entry: - domain = entry.lower() - if whitelist_pattern.match(entry): - file_stats['skip'] += 1 - else: - if domain_pattern.match(domain): - file_stats['blocklist'] += 1 - blocklist_items['data'][entry] = {"bl": bl_shortcode} - else: + if not skip_download: + # fetch all blocklists, will replace the existing file used by Unbound + if cnf.has_section('settings'): + if cnf.has_option('settings', 'address'): + blocklist_items['config']['dst_addr'] = cnf.get('settings', 'address') + if cnf.has_option('settings', 'rcode'): + blocklist_items['config']['rcode'] = cnf.get('settings', 'rcode') + if cnf.has_section('blocklists'): + for blocklist in cnf['blocklists']: + list_type = blocklist.split('_', 1) + bl_shortcode = 'Custom' if list_type[0] == 'custom' else list_type[1] + file_stats = {'uri': cnf['blocklists'][blocklist], 'skip' : 0, 'blocklist': 0, 'lines' :0} + for line in uri_reader(cnf['blocklists'][blocklist]): + file_stats['lines'] += 1 + # cut line into parts before comment marker (if any) + tmp = line.split('#')[0].split() + entry = None + while tmp: + entry = tmp.pop(-1) + if entry not in ['127.0.0.1', '0.0.0.0']: + break + if entry: + domain = entry.lower() + if whitelist_pattern.match(entry): file_stats['skip'] += 1 + else: + if domain_pattern.match(domain): + file_stats['blocklist'] += 1 + blocklist_items['data'][entry] = {"bl": bl_shortcode} + else: + file_stats['skip'] += 1 - syslog.syslog( - syslog.LOG_NOTICE, - 'blocklist download %(uri)s (lines: %(lines)d exclude: %(skip)d block: %(blocklist)d)' % file_stats - ) + syslog.syslog( + syslog.LOG_NOTICE, + 'blocklist download %(uri)s (lines: %(lines)d exclude: %(skip)d block: %(blocklist)d)' % file_stats + ) + + # after a download, always apply exact custom matches on top of it + if cnf.has_section('include'): + for item in cnf['include']: + entry = cnf['include'][item].rstrip().lower() + if domain_pattern.match(entry): + blocklist_items['data'][entry] = {"bl": "Custom"} + + else: + # only modify the existing list, administrate on added and removed exact custom matches + syslog.syslog(syslog.LOG_NOTICE, 'blocklist: skip download') + if (diffs_added['include'] or diffs_removed['include']) and os.path.exists('/var/unbound/data/dnsbl.json'): + blocklist_items = ujson.load(open('/var/unbound/data/dnsbl.json', 'r')) + for item in diffs_removed['include']: + del blocklist_items['data'][item[1].rstrip().lower()] + for item in diffs_added['include']: + blocklist_items['data'][item[1].rstrip().lower()] = {"bl": "Custom"} + + with open('/tmp/unbound-blocklists.conf.cache', 'w') as cache_config: + # cache the current config so we can diff on it the next time + cnf.write(cache_config) # write out results if not os.path.exists('/var/unbound/data'): os.makedirs('/var/unbound/data') with open("/var/unbound/data/dnsbl.json.new", 'w') as unbound_outf: if blocklist_items: - json.dump(blocklist_items, unbound_outf) + ujson.dump(blocklist_items, unbound_outf) # atomically replace the current dnsbl so unbound can pick up on it os.replace('/var/unbound/data/dnsbl.json.new', '/var/unbound/data/dnsbl.json') diff --git a/src/opnsense/service/templates/OPNsense/Unbound/core/blocklists.conf b/src/opnsense/service/templates/OPNsense/Unbound/core/blocklists.conf index 3ba0089fa..47d12297b 100644 --- a/src/opnsense/service/templates/OPNsense/Unbound/core/blocklists.conf +++ b/src/opnsense/service/templates/OPNsense/Unbound/core/blocklists.conf @@ -69,4 +69,12 @@ custom_pattern_{{loop.index}}={{ pattern }} {% endfor %} {% endif %} +[include] +{% if not helpers.empty('OPNsense.unboundplus.dnsbl.blocklists')%} +# user defined +{% for pattern in OPNsense.unboundplus.dnsbl.blocklists.split(',') %} +custom_pattern_{{loop.index}}={{ pattern }} +{% endfor %} +{% endif %} + {% endif %}