mirror of
https://github.com/lucaspalomodevelop/core.git
synced 2026-03-17 18:14:42 +00:00
Unbound / Blocklists: add exact domain blocking and integrate into overview page (#6205)
This include the ability to whitelist it from the same page as well. Relevant to both the top passed/blocked domains, as well as the detailed query grid. blocklists.py has been modified in such a way that it will detect whether it needs to start the download process or simply administrate locally. The latter currently only happens when custom domains for blocking have been added/removed by a user. The reasoning is that we can easily extend/shrink the current blocklist when it comes to blocking exact domains as this is handled on the incoming side. However, while we can modify the current list to accomodate a new whitelist entry (which can be regex), we (currently) cannot know which domains were skipped in the process of retrieving them in the first place if a user explicitly removes a whitelist entry. Therefore we decide to re-run the download on a whitelist action. furthermore, the updateBlocklistAction in the controller administrates how the model is updated (e.g. when a blocked item is whitelisted, it should be removed from the blocklist model entry and added to the whitelist) In the future we could optimize the whole process by checking if a remote file has changed in date or size.
This commit is contained in:
Stephan de Wit
GitHub
parent
7723ccaf86
commit
e5d6acd2eb
@ -40,6 +40,58 @@ class SettingsController extends ApiMutableModelControllerBase
|
||||
|
||||
private $type = 'dot';
|
||||
|
||||
public function updateBlocklistAction()
|
||||
{
|
||||
$result = ["status" => "failed"];
|
||||
if ($this->request->isPost() && $this->request->hasPost('domain') && $this->request->hasPost('type')) {
|
||||
Config::getInstance()->lock();
|
||||
$domain = $this->request->getPost('domain');
|
||||
$type = $this->request->getPost('type');
|
||||
$mdl = $this->getModel();
|
||||
$item = $mdl->getNodeByReference('dnsbl.'.$type);
|
||||
|
||||
if ($item != null) {
|
||||
$remove = function($csv, $item) {
|
||||
$parts = explode(',', $csv);
|
||||
while(($i = array_search($item, $parts)) !== false) {
|
||||
unset($parts[$i]);
|
||||
}
|
||||
return implode(',', $parts);
|
||||
};
|
||||
|
||||
// strip off any trailing dot
|
||||
$value = rtrim($domain, '.');
|
||||
$wl = (string)$mdl->dnsbl->whitelists;
|
||||
$bl = (string)$mdl->dnsbl->blocklists;
|
||||
|
||||
if (strpos((string)$mdl->dnsbl->$type, $value) !== false) {
|
||||
// value already in model, no need to re-run a potentially
|
||||
// expensive dnsbl action
|
||||
return ["status" => "OK"];
|
||||
}
|
||||
|
||||
// Check if domains should be switched around in the model
|
||||
if ($type == 'whitelists' && strpos($bl, $value) !== false) {
|
||||
$mdl->dnsbl->blocklists = $remove((string)$mdl->dnsbl->blocklists, $value);
|
||||
} elseif ($type == 'blocklists' && strpos($wl, $value) !== false) {
|
||||
$mdl->dnsbl->whitelists = $remove((string)$mdl->dnsbl->whitelists, $value);
|
||||
}
|
||||
|
||||
// update the model
|
||||
$list = array_filter(explode(',', (string)$item));
|
||||
$list[] = $value;
|
||||
$mdl->dnsbl->$type = implode(',', $list);
|
||||
|
||||
$mdl->serializeToConfig();
|
||||
Config::getInstance()->save();
|
||||
|
||||
$service = new \OPNsense\Unbound\Api\ServiceController();
|
||||
$result = $service->dnsblAction();
|
||||
}
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
public function getNameserversAction()
|
||||
{
|
||||
if ($this->request->isGet()) {
|
||||
|
||||
@ -28,6 +28,14 @@
|
||||
<allownew>true</allownew>
|
||||
<help>List of domains to whitelist. You can use regular expressions.</help>
|
||||
</field>
|
||||
<field>
|
||||
<id>unbound.dnsbl.blocklists</id>
|
||||
<label>Blocklist Domains</label>
|
||||
<type>select_multiple</type>
|
||||
<style>tokenize</style>
|
||||
<allownew>true</allownew>
|
||||
<help>List of domains to blocklist. Only exact matches are supported.</help>
|
||||
</field>
|
||||
<field>
|
||||
<id>unbound.dnsbl.address</id>
|
||||
<label>Destination Address</label>
|
||||
|
||||
@ -152,6 +152,9 @@
|
||||
<whitelists type="CSVListField">
|
||||
<Required>N</Required>
|
||||
</whitelists>
|
||||
<blocklists type="CSVListField">
|
||||
<Required>N</Required>
|
||||
</blocklists>
|
||||
<address type="NetworkField">
|
||||
<Required>N</Required>
|
||||
<NetMaskAllowed>N</NetMaskAllowed>
|
||||
|
||||
@ -347,32 +347,37 @@
|
||||
});
|
||||
}
|
||||
|
||||
function createTopList(id, data) {
|
||||
function createTopList(id, data, type) {
|
||||
let idx = 1;
|
||||
for (const [domain, statObj] of Object.entries(data)) {
|
||||
let class_type = type == "pass" ? "block-domain" : "whitelist-domain";
|
||||
let icon_type = type == "pass" ? "fa fa-ban text-danger" : "fa fa-pencil text-info";
|
||||
let bl = statObj.hasOwnProperty('blocklist') ? '(' + statObj.blocklist + ')' : '';
|
||||
$('#' + id).append(
|
||||
'<li class="list-group-item list-group-item-border">' +
|
||||
'<li class="list-group-item list-group-item-border top-item">' +
|
||||
idx + '. ' + domain + ' ' + bl +
|
||||
'<span class="counter">'+ statObj.total +' (' + statObj.pcnt +'%)</span>' +
|
||||
'</li>'
|
||||
'<span class="counter">'+ statObj.total +' (' + statObj.pcnt +'%)' +
|
||||
'<button class="'+ class_type + '" data-value="'+ domain +'" ' +
|
||||
'data-toggle="tooltip" style="margin-left: 10px;"><i class="' + icon_type + '"></i></button>' +
|
||||
'</span></li>'
|
||||
)
|
||||
idx++;
|
||||
}
|
||||
$(".block-domain").tooltip().attr('title', "{{ lang._('Block Domain') }}");
|
||||
$(".whitelist-domain").tooltip().attr('title', "{{ lang._('Whitelist Domain') }}");
|
||||
}
|
||||
|
||||
function create_or_update_totals() {
|
||||
ajaxGet('/api/unbound/overview/totals/10', {}, function(data, status) {
|
||||
$('#top').not(':first').empty();
|
||||
$('#top-blocked').not(':first').empty();
|
||||
$('.top-item').remove();
|
||||
|
||||
$('#totalCounter').html(data.total);
|
||||
$('#blockedCounter').html(data.blocked.total + " (" + data.blocked.pcnt + "%)");
|
||||
$('#sizeCounter').html(data.blocklist_size);
|
||||
$('#resolvedCounter').html(data.resolved.total + " (" + data.resolved.pcnt + "%)");
|
||||
|
||||
createTopList('top', data.top);
|
||||
createTopList('top-blocked', data.top_blocked);
|
||||
createTopList('top', data.top, 'pass');
|
||||
createTopList('top-blocked', data.top_blocked, 'block');
|
||||
|
||||
$('#top li:nth-child(even)').addClass('odd-bg');
|
||||
$('#top-blocked li:nth-child(even)').addClass('odd-bg');
|
||||
@ -445,6 +450,26 @@
|
||||
updateClientChart(this.checked);
|
||||
})
|
||||
|
||||
$(document).on('click', '.block-domain', function() {
|
||||
$(this).remove("i").html('<i class="fa fa-spinner fa-spin"></i>');
|
||||
ajaxCall('/api/unbound/settings/updateBlocklist', {
|
||||
'domain': $(this).data('value'),
|
||||
'type': 'blocklists'
|
||||
}, function(data, status) {
|
||||
$(".block-domain").remove("i").html('<i class="fa fa-ban text-danger"></i>');
|
||||
});
|
||||
});
|
||||
|
||||
$(document).on('click', '.whitelist-domain', function() {
|
||||
$(this).remove("i").html('<i class="fa fa-spinner fa-spin"></i>');
|
||||
ajaxCall('/api/unbound/settings/updateBlocklist', {
|
||||
'domain': $(this).data('value'),
|
||||
'type': 'whitelists'
|
||||
}, function(data, status) {
|
||||
$(".whitelist-domain").remove("i").html('<i class="fa fa-pencil text-info"></i>');
|
||||
});
|
||||
});
|
||||
|
||||
do_startup().done(function() {
|
||||
$('.wrapper').show();
|
||||
}).fail(function() {
|
||||
@ -467,7 +492,16 @@
|
||||
},
|
||||
"resolveformatter": function (column, row) {
|
||||
return row.resolve_time_ms + 'ms';
|
||||
}
|
||||
},
|
||||
"commands": function (column, row) {
|
||||
let btn = '';
|
||||
if (row.action == 'Pass') {
|
||||
btn = '<button type="button" class="btn-secondary block-domain" data-domain=' + row.domain + ' data-toggle="tooltip"><i class="fa fa-ban text-danger"></i></button> ';
|
||||
} else if (row.action == 'Block') {
|
||||
btn = '<button type="button" class="btn-secondary whitelist-domain" data-domain=' + row.domain + ' data-toggle="tooltip"><i class="fa fa-pencil text-info"></i></button>';
|
||||
}
|
||||
return btn;
|
||||
},
|
||||
},
|
||||
statusMapping: {
|
||||
0: "query-success",
|
||||
@ -477,6 +511,30 @@
|
||||
4: "danger"
|
||||
}
|
||||
}
|
||||
}).on("loaded.rs.jquery.bootgrid", function (e) {
|
||||
$(".block-domain").tooltip().attr('title', "{{ lang._('Block Domain') }}");
|
||||
$(".whitelist-domain").tooltip().attr('title', "{{ lang._('Whitelist Domain') }}");
|
||||
grid_queries.find(".block-domain").on("click", function(e) {
|
||||
$(this).remove("i").html('<i class="fa fa-spinner fa-spin"></i>');
|
||||
ajaxCall('/api/unbound/settings/updateBlocklist', {
|
||||
'domain': $(this).data('domain'),
|
||||
'type': 'blocklists',
|
||||
}, function(data, status) {
|
||||
let btn = grid_queries.find(".block-domain");
|
||||
btn.remove("i").html('<i class="fa fa-ban text-danger"></i>');
|
||||
});
|
||||
});
|
||||
|
||||
grid_queries.find(".whitelist-domain").on("click", function(e) {
|
||||
$(this).remove("i").html('<i class="fa fa-spinner fa-spin"></i>');
|
||||
ajaxCall('/api/unbound/settings/updateBlocklist', {
|
||||
'domain': $(this).data('domain'),
|
||||
'type': 'whitelists',
|
||||
}, function(data, status) {
|
||||
let btn = grid_queries.find(".whitelist-domain");
|
||||
btn.remove("i").html('<i class="fa fa-pencil text-info"></i>');
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
if (e.target.id == 'query_overview_tab') {
|
||||
@ -569,7 +627,7 @@
|
||||
<div class="col-md-2"></div>
|
||||
<div class="col-md-2">
|
||||
<div class="vertical-center">
|
||||
<label class="h-100" style="margin-right: 5px;">Logarithmic</label>
|
||||
<label class="h-100" style="margin-right: 5px;">{{ lang._('Logarithmic') }}</label>
|
||||
<input id="toggle-log-qchart" type="checkbox"></input>
|
||||
</div>
|
||||
</div>
|
||||
@ -658,6 +716,7 @@
|
||||
<th data-column-id="resolve_time_ms" data-type="string" data-formatter="resolveformatter">{{ lang._('Resolve time') }}</th>
|
||||
<th data-column-id="ttl" data-type="string">{{ lang._('TTL') }}</th>
|
||||
<th data-column-id="blocklist" data-type="string">{{ lang._('Blocklist') }}</th>
|
||||
<th data-column-id="commands" data-width="7em" data-formatter="commands" data-sortable="false">{{ lang._('Command') }}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
|
||||
@ -35,7 +35,7 @@ import time
|
||||
import fcntl
|
||||
from configparser import ConfigParser
|
||||
import requests
|
||||
import json
|
||||
import ujson
|
||||
|
||||
def uri_reader(uri):
|
||||
req_opts = {
|
||||
@ -76,8 +76,6 @@ def uri_reader(uri):
|
||||
'blocklist download : unable to download file from %s (status_code: %d)' % (uri, req.status_code)
|
||||
)
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# check for a running download process, this may take a while so it's better to check...
|
||||
try:
|
||||
@ -100,9 +98,33 @@ if __name__ == '__main__':
|
||||
'data': {},
|
||||
'config': {}
|
||||
}
|
||||
skip_download = False
|
||||
if os.path.exists('/tmp/unbound-blocklists.conf'):
|
||||
cnf = ConfigParser()
|
||||
cnf.read('/tmp/unbound-blocklists.conf')
|
||||
|
||||
cnf_cache = ConfigParser()
|
||||
if os.path.exists('/tmp/unbound-blocklists.conf.cache'):
|
||||
cnf_cache.read('/tmp/unbound-blocklists.conf.cache')
|
||||
else:
|
||||
cnf_cache.read('/tmp/unbound-blocklists.conf')
|
||||
|
||||
if cnf.sections() and cnf_cache.sections():
|
||||
# get the difference between the old and new configuration, there won't be any
|
||||
# if we're starting up, so it will proceed as normal.
|
||||
diff_cnf = {d: set(map(tuple, v.items())) for d,v in cnf._sections.items()}
|
||||
diff_cnf_cache = {d: set(map(tuple, v.items())) for d,v in cnf_cache._sections.items()}
|
||||
diffs_added = {header: diff_cnf[header] - diff_cnf_cache[header] for header, _ in diff_cnf.items()}
|
||||
diffs_removed = {header: diff_cnf_cache[header] - diff_cnf[header] for header, _ in diff_cnf.items()}
|
||||
|
||||
# we can only skip download if the include option has changed, but it must proceed
|
||||
# if any other option has changed
|
||||
if (diffs_added['include'] or diffs_removed['include']):
|
||||
skip_download = True
|
||||
for (a, r) in zip(diffs_added, diffs_removed):
|
||||
if (a != 'include' and r != 'include') and (diffs_added[a] or diffs_removed[r]):
|
||||
skip_download = False
|
||||
|
||||
# exclude (white) lists, compile to regex to be used to filter blocklist entries
|
||||
if cnf.has_section('exclude'):
|
||||
exclude_list = set()
|
||||
@ -123,48 +145,70 @@ if __name__ == '__main__':
|
||||
whitelist_pattern = re.compile(wp, re.IGNORECASE)
|
||||
syslog.syslog(syslog.LOG_NOTICE, 'blocklist download : exclude domains matching %s' % wp)
|
||||
|
||||
# fetch all blocklists
|
||||
if cnf.has_section('settings'):
|
||||
if cnf.has_option('settings', 'address'):
|
||||
blocklist_items['config']['dst_addr'] = cnf.get('settings', 'address')
|
||||
if cnf.has_option('settings', 'rcode'):
|
||||
blocklist_items['config']['rcode'] = cnf.get('settings', 'rcode')
|
||||
if cnf.has_section('blocklists'):
|
||||
for blocklist in cnf['blocklists']:
|
||||
list_type = blocklist.split('_', 1)
|
||||
bl_shortcode = 'Custom' if list_type[0] == 'custom' else list_type[1]
|
||||
file_stats = {'uri': cnf['blocklists'][blocklist], 'skip' : 0, 'blocklist': 0, 'lines' :0}
|
||||
for line in uri_reader(cnf['blocklists'][blocklist]):
|
||||
file_stats['lines'] += 1
|
||||
# cut line into parts before comment marker (if any)
|
||||
tmp = line.split('#')[0].split()
|
||||
entry = None
|
||||
while tmp:
|
||||
entry = tmp.pop(-1)
|
||||
if entry not in ['127.0.0.1', '0.0.0.0']:
|
||||
break
|
||||
if entry:
|
||||
domain = entry.lower()
|
||||
if whitelist_pattern.match(entry):
|
||||
file_stats['skip'] += 1
|
||||
else:
|
||||
if domain_pattern.match(domain):
|
||||
file_stats['blocklist'] += 1
|
||||
blocklist_items['data'][entry] = {"bl": bl_shortcode}
|
||||
else:
|
||||
if not skip_download:
|
||||
# fetch all blocklists, will replace the existing file used by Unbound
|
||||
if cnf.has_section('settings'):
|
||||
if cnf.has_option('settings', 'address'):
|
||||
blocklist_items['config']['dst_addr'] = cnf.get('settings', 'address')
|
||||
if cnf.has_option('settings', 'rcode'):
|
||||
blocklist_items['config']['rcode'] = cnf.get('settings', 'rcode')
|
||||
if cnf.has_section('blocklists'):
|
||||
for blocklist in cnf['blocklists']:
|
||||
list_type = blocklist.split('_', 1)
|
||||
bl_shortcode = 'Custom' if list_type[0] == 'custom' else list_type[1]
|
||||
file_stats = {'uri': cnf['blocklists'][blocklist], 'skip' : 0, 'blocklist': 0, 'lines' :0}
|
||||
for line in uri_reader(cnf['blocklists'][blocklist]):
|
||||
file_stats['lines'] += 1
|
||||
# cut line into parts before comment marker (if any)
|
||||
tmp = line.split('#')[0].split()
|
||||
entry = None
|
||||
while tmp:
|
||||
entry = tmp.pop(-1)
|
||||
if entry not in ['127.0.0.1', '0.0.0.0']:
|
||||
break
|
||||
if entry:
|
||||
domain = entry.lower()
|
||||
if whitelist_pattern.match(entry):
|
||||
file_stats['skip'] += 1
|
||||
else:
|
||||
if domain_pattern.match(domain):
|
||||
file_stats['blocklist'] += 1
|
||||
blocklist_items['data'][entry] = {"bl": bl_shortcode}
|
||||
else:
|
||||
file_stats['skip'] += 1
|
||||
|
||||
syslog.syslog(
|
||||
syslog.LOG_NOTICE,
|
||||
'blocklist download %(uri)s (lines: %(lines)d exclude: %(skip)d block: %(blocklist)d)' % file_stats
|
||||
)
|
||||
syslog.syslog(
|
||||
syslog.LOG_NOTICE,
|
||||
'blocklist download %(uri)s (lines: %(lines)d exclude: %(skip)d block: %(blocklist)d)' % file_stats
|
||||
)
|
||||
|
||||
# after a download, always apply exact custom matches on top of it
|
||||
if cnf.has_section('include'):
|
||||
for item in cnf['include']:
|
||||
entry = cnf['include'][item].rstrip().lower()
|
||||
if domain_pattern.match(entry):
|
||||
blocklist_items['data'][entry] = {"bl": "Custom"}
|
||||
|
||||
else:
|
||||
# only modify the existing list, administrate on added and removed exact custom matches
|
||||
syslog.syslog(syslog.LOG_NOTICE, 'blocklist: skip download')
|
||||
if (diffs_added['include'] or diffs_removed['include']) and os.path.exists('/var/unbound/data/dnsbl.json'):
|
||||
blocklist_items = ujson.load(open('/var/unbound/data/dnsbl.json', 'r'))
|
||||
for item in diffs_removed['include']:
|
||||
del blocklist_items['data'][item[1].rstrip().lower()]
|
||||
for item in diffs_added['include']:
|
||||
blocklist_items['data'][item[1].rstrip().lower()] = {"bl": "Custom"}
|
||||
|
||||
with open('/tmp/unbound-blocklists.conf.cache', 'w') as cache_config:
|
||||
# cache the current config so we can diff on it the next time
|
||||
cnf.write(cache_config)
|
||||
|
||||
# write out results
|
||||
if not os.path.exists('/var/unbound/data'):
|
||||
os.makedirs('/var/unbound/data')
|
||||
with open("/var/unbound/data/dnsbl.json.new", 'w') as unbound_outf:
|
||||
if blocklist_items:
|
||||
json.dump(blocklist_items, unbound_outf)
|
||||
ujson.dump(blocklist_items, unbound_outf)
|
||||
|
||||
# atomically replace the current dnsbl so unbound can pick up on it
|
||||
os.replace('/var/unbound/data/dnsbl.json.new', '/var/unbound/data/dnsbl.json')
|
||||
|
||||
@ -69,4 +69,12 @@ custom_pattern_{{loop.index}}={{ pattern }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
[include]
|
||||
{% if not helpers.empty('OPNsense.unboundplus.dnsbl.blocklists')%}
|
||||
# user defined
|
||||
{% for pattern in OPNsense.unboundplus.dnsbl.blocklists.split(',') %}
|
||||
custom_pattern_{{loop.index}}={{ pattern }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
{% endif %}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user