Unbound: integrate endpoints, but leave model in place to avoid migration issues and force de-install of the plugin.

This commit is contained in:
Ad Schellevis 2020-04-09 14:22:54 +02:00 committed by Ad Schellevis
parent f93844748a
commit e330bb46f2
23 changed files with 98 additions and 286 deletions

View File

@ -2,6 +2,7 @@
/*
* Copyright (C) 2019 Michael Muenz <m.muenz@gmail.com>
* Copyright (C) 2020 Deciso B.V.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -26,7 +27,7 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
namespace OPNsense\Unboundplus\Api;
namespace OPNsense\Unbound\Api;
use OPNsense\Base\ApiMutableModelControllerBase;

View File

@ -2,6 +2,7 @@
/*
* Copyright (C) 2019 Michael Muenz <m.muenz@gmail.com>
* Copyright (C) 2020 Deciso B.V.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -26,7 +27,7 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
namespace OPNsense\Unboundplus\Api;
namespace OPNsense\Unbound\Api;
use OPNsense\Base\ApiMutableModelControllerBase;

View File

@ -2,6 +2,7 @@
/*
* Copyright (C) 2019 Michael Muenz <m.muenz@gmail.com>
* Copyright (C) 2020 Deciso B.V.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -26,7 +27,7 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
namespace OPNsense\Unboundplus\Api;
namespace OPNsense\Unbound\Api;
use OPNsense\Base\ApiMutableServiceControllerBase;
use OPNsense\Core\Backend;
@ -36,7 +37,7 @@ use OPNsense\Unboundplus\Miscellaneous;
class ServiceController extends ApiMutableServiceControllerBase
{
protected static $internalServiceClass = '\OPNsense\Unboundplus\Dnsbl';
protected static $internalServiceTemplate = 'OPNsense/Unboundplus';
protected static $internalServiceTemplate = 'OPNsense/Unbound';
protected static $internalServiceEnabled = 'enabled';
protected static $internalServiceName = 'unboundplus';
@ -45,18 +46,8 @@ class ServiceController extends ApiMutableServiceControllerBase
$this->sessionClose();
$mdl = new Dnsbl();
$backend = new Backend();
$backend->configdRun('template reload OPNsense/Unboundplus');
$response = $backend->configdpRun('unboundplus dnsbl', array((string)$mdl->type));
return array("response" => $response);
}
public function reloadunboundAction()
{
$this->sessionClose();
$mdl = new Miscellaneous();
$backend = new Backend();
$backend->configdRun('template reload OPNsense/Unboundplus');
$response = $backend->configdpRun('unbound reload', array((string)$mdl->type));
return array("response" => $response);
$backend->configdRun('template reload OPNsense/Unbound');
$response = $backend->configdpRun('unbound dnsbl', array((string)$mdl->type));
return array("status" => $response);
}
}

View File

@ -26,13 +26,15 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
namespace OPNsense\Unboundplus;
namespace OPNsense\Unbound;
class DnsblController extends \OPNsense\Base\IndexController
use OPNsense\Base\IndexController;
class DnsblController extends IndexController
{
public function indexAction()
{
$this->view->dnsblForm = $this->getForm('dnsbl');
$this->view->pick('OPNsense/Unboundplus/dnsbl');
$this->view->pick('OPNsense/Unbound/dnsbl');
}
}

View File

@ -26,13 +26,15 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
namespace OPNsense\Unboundplus;
namespace OPNsense\Unbound;
class MiscellaneousController extends \OPNsense\Base\IndexController
use OPNsense\Base\IndexController;
class MiscellaneousController extends IndexController
{
public function indexAction()
{
$this->view->miscellaneousForm = $this->getForm('miscellaneous');
$this->view->pick('OPNsense/Unboundplus/miscellaneous');
$this->view->pick('OPNsense/Unbound/miscellaneous');
}
}

View File

@ -38,4 +38,11 @@
<pattern>api/diagnostics/log/core/resolver/*</pattern>
</patterns>
</page-diagnostics-logs-resolver>
<page-services-unbound>
<name>Services: Unbound (mvc)</name>
<patterns>
<pattern>ui/unbound/*</pattern>
<pattern>api/unbound/*</pattern>
</patterns>
</page-services-unbound>
</acl>

View File

@ -12,6 +12,8 @@
</ACL>
<Statistics order="50" url="/ui/unbound/stats"/>
<LogFile VisibleName="Log File" order="100" url="/ui/diagnostics/log/core/resolver"/>
<Blacklist order="50" url="/ui/unbound/dnsbl/index"/>
<Miscellaneous order="60" url="/ui/unbound/miscellaneous/index"/>
</Unbound>
</Services>
</menu>

View File

@ -1,9 +0,0 @@
<acl>
<page-services-unbound>
<name>Services: Unbound DNSBL</name>
<patterns>
<pattern>ui/unboundplus/*</pattern>
<pattern>api/unboundplus/*</pattern>
</patterns>
</page-services-unbound>
</acl>

View File

@ -1,8 +0,0 @@
<menu>
<Services>
<Unbound>
<Blacklist order="50" url="/ui/unboundplus/dnsbl/index"/>
<Miscellaneous order="60" url="/ui/unboundplus/miscellaneous/index"/>
</Unbound>
</Services>
</menu>

View File

@ -25,30 +25,35 @@
# POSSIBILITY OF SUCH DAMAGE.
#}
<script>
$(document).ready(function() {
var data_get_map = {'frm_dnsbl_settings':"/api/unbound/dnsbl/get"};
mapDataToFormUI(data_get_map).done(function(data){
formatTokenizersUI();
$('.selectpicker').selectpicker('refresh');
});
$("#saveAct").SimpleActionButton({
onPreAction: function() {
const dfObj = new $.Deferred();
saveFormToEndpoint("/api/unbound/dnsbl/set", 'frm_dnsbl_settings', function(){
dfObj.resolve();
});
return dfObj;
}
});
});
</script>
<div class="content-box" style="padding-bottom: 1.5em;">
{{ partial("layout_partials/base_form",['fields':dnsblForm,'id':'frm_dnsbl_settings'])}}
<div class="col-md-12">
<hr />
<button class="btn btn-primary" id="saveAct" type="button"><b>{{ lang._('Save') }}</b> <i id="saveAct_progress"></i></button>
<button class="btn btn-primary" id="saveAct"
data-endpoint='/api/unbound/service/dnsbl'
data-label="{{ lang._('Save') }}"
data-error-title="{{ lang._('Error reloading unbound') }}"
type="button">
</button>
</div>
</div>
<script>
$(function() {
var data_get_map = {'frm_dnsbl_settings':"/api/unboundplus/dnsbl/get"};
mapDataToFormUI(data_get_map).done(function(data){
formatTokenizersUI();
$('.selectpicker').selectpicker('refresh');
});
// link save button to API set action
$("#saveAct").click(function(){
saveFormToEndpoint(url="/api/unboundplus/dnsbl/set", formid='frm_dnsbl_settings',callback_ok=function(){
$("#saveAct_progress").addClass("fa fa-spinner fa-pulse");
ajaxCall(url="/api/unboundplus/service/dnsbl", sendData={}, callback=function(data,status) {
$("#saveAct_progress").removeClass("fa fa-spinner fa-pulse");
});
});
});
});
</script>

View File

@ -1,5 +1,5 @@
{#
# Copyright (c) 2019 Deciso B.V.
# Copyright (c) 2019-2020 Deciso B.V.
# Copyright (c) 2019 Michael Muenz <m.muenz@gmail.com>
# All rights reserved.
#
@ -25,30 +25,34 @@
# POSSIBILITY OF SUCH DAMAGE.
#}
<script>
$(document).ready(function() {
var data_get_map = {'frm_miscellaneous_settings':"/api/unbound/miscellaneous/get"};
mapDataToFormUI(data_get_map).done(function(data){
formatTokenizersUI();
$('.selectpicker').selectpicker('refresh');
});
$("#saveAct").SimpleActionButton({
onPreAction: function() {
const dfObj = new $.Deferred();
saveFormToEndpoint("/api/unbound/miscellaneous/set", 'frm_miscellaneous_settings', function(){
dfObj.resolve();
});
return dfObj;
}
});
});
</script>
<div class="content-box" style="padding-bottom: 1.5em;">
{{ partial("layout_partials/base_form",['fields':miscellaneousForm,'id':'frm_miscellaneous_settings'])}}
<div class="col-md-12">
<hr />
<button class="btn btn-primary" id="saveAct" type="button"><b>{{ lang._('Save') }}</b> <i id="saveAct_progress"></i></button>
<button class="btn btn-primary" id="saveAct"
data-endpoint='/api/unbound/service/reload'
data-label="{{ lang._('Save') }}"
data-error-title="{{ lang._('Error reloading unbound') }}"
type="button">
</button>
</div>
</div>
<script>
$(function() {
var data_get_map = {'frm_miscellaneous_settings':"/api/unboundplus/miscellaneous/get"};
mapDataToFormUI(data_get_map).done(function(data){
formatTokenizersUI();
$('.selectpicker').selectpicker('refresh');
});
// link save button to API set action
$("#saveAct").click(function(){
saveFormToEndpoint(url="/api/unboundplus/miscellaneous/set", formid='frm_miscellaneous_settings',callback_ok=function(){
$("#saveAct_progress").addClass("fa fa-spinner fa-pulse");
ajaxCall(url="/api/unboundplus/service/reloadunbound", sendData={}, callback=function(data,status) {
$("#saveAct_progress").removeClass("fa fa-spinner fa-pulse");
});
});
});
});
</script>

View File

@ -1,197 +0,0 @@
#!/usr/local/bin/python3
# DNS BL script
# Copyright 2020 Petr Kejval <petr.kejval6@gmail.com>
# Downloads blacklisted domains from user specified URLs and "compile" them into unbound.conf compatible file
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
import re, urllib3, threading, subprocess
re_blacklist = re.compile(r'(^127\.0\.0\.1[\s]+|^0\.0\.0\.0[\s]+)([0-9a-z_.-]+)(?:\s|$)|^([0-9a-z_.-]+)(?:\s|$)', re.I)
re_whitelist = re.compile(r'$^') # default - match nothing
blacklist = set()
urls = set()
predefined_lists = {
"aa": "https://adaway.org/hosts.txt",
"ag": "https://justdomains.github.io/blocklists/lists/adguarddns-justdomains.txt",
"bla": "https://blocklist.site/app/dl/ads",
"blf": "https://blocklist.site/app/dl/fraud",
"blp": "https://blocklist.site/app/dl/phishing",
"ca": "http://sysctl.org/cameleon/hosts",
"el": "https://justdomains.github.io/blocklists/lists/easylist-justdomains.txt",
"ep": "https://justdomains.github.io/blocklists/lists/easyprivacy-justdomains.txt",
"emd": "https://hosts-file.net/emd.txt",
"hpa": "https://hosts-file.net/ad_servers.txt",
"hpf": "https://hosts-file.net/fsa.txt",
"hpp": "https://hosts-file.net/psh.txt",
"hup": "https://hosts-file.net/pup.txt",
"nc": "https://justdomains.github.io/blocklists/lists/nocoin-justdomains.txt",
"rw": "https://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt",
"mw": "http://malwaredomains.lehigh.edu/files/justdomains",
"pa": "https://raw.githubusercontent.com/chadmayfield/my-pihole-blocklists/master/lists/pi_blocklist_porn_all.list",
"pt": "https://raw.githubusercontent.com/chadmayfield/pihole-blocklists/master/lists/pi_blocklist_porn_top1m.list",
"sa": "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt",
"sb": "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts",
"st": "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt",
"ws": "https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt",
"wsu": "https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/update.txt",
"wse": "https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/extra.txt",
"yy": "http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&mimetype=plaintext"
}
def add_to_blacklist(domain):
""" Checks if domain is present in whitelist. If not, domain is addded to BL set. """
match = re_whitelist.match(domain)
if not match:
blacklist.add(domain)
def parse_line(line):
""" Checks if line matches re_blacklist. If so, tries add domain to BL set. """
global blacklist
line = line.replace('\\t', " ")
line = line.replace('\\r', "")
match = re_blacklist.match(line)
if match:
if match.group(2) != None:
add_to_blacklist(match.group(2))
elif match.group(3) != None:
add_to_blacklist(match.group(3))
def process_url(url):
""" Reads and parses blacklisted domains from URL into BL set. """
print(f"Processing BL items from: {url}")
try:
http = urllib3.PoolManager(timeout=5.0)
r = http.request('GET', url, retries=2)
if r.status == 200:
for line in str(r.data).split('\\n'):
parse_line(line)
except Exception as e:
print(str(e))
def save_config_file():
""" Saves blacklist in unbound.conf format """
print(f"Saving {len(blacklist)} blacklisted domains into dnsbl.conf")
try:
with open("/var/unbound/etc/dnsbl.conf", 'w') as file:
# No domains found or DNSBL is disabled
if (len(blacklist) == 0):
file.write("")
else:
file.write('server:\n')
for line in blacklist:
#file.write('local-zone: "' + str(line) + '" static\n')
file.write('local-data: "' + str(line) + ' A 0.0.0.0"\n')
except Exception as e:
print(str(e))
exit(1)
def load_list(path, separator=None):
""" Reads file with specified path into set to ensure unique values.
Splits lines with defined separator. If sperator==None no split is performed. """
result = set()
try:
with open(path, 'r') as file:
for line in file.readlines():
if not separator == None:
for element in line.split(separator):
result.add(element.replace('\n', ''))
else:
result.add(line.replace('\n', ''))
except Exception as e:
print(str(e))
return result
def load_whitelist():
""" Loads user defined whitelist in regex format and compiles it. """
print("Loading whitelist")
global re_whitelist
wl = load_list('/var/unbound/etc/whitelist.inc', ',')
wl.add(r'.*localhost$')
wl.add(r'^(?![a-zA-Z\d]).*') # Exclude domains NOT starting with alphanumeric char
print(f"Loaded {len(wl)} whitelist items")
try:
re_whitelist = re.compile('|'.join(wl), re.I)
except Exception as e:
print(f"Whitelist regex compile failed: {str(e)}")
def load_blacklists():
""" Loads user defined blacklists URLs. """
print("Loading blacklists URLs")
global urls
urls = load_list('/var/unbound/etc/lists.inc', ',')
print(f"Loaded {len(urls)} blacklists URLs")
def load_predefined_lists():
""" Loads user chosen predefined lists """
print("Loading predefined lists URLs")
global urls
lists = load_list('/var/unbound/etc/dnsbl.inc')
types = set()
for first in lists:
first = str(first).split('=')[1]
first = str(first).replace('"', '').replace('\n', '')
first = first.split(',')
for type in first:
types.add(type)
break
print(f"Loaded {len(types)} predefined blacklists URLs")
for type in types:
try:
urls.add(predefined_lists[type])
except KeyError:
continue
except Exception as e:
print(str(e))
if __name__ == "__main__":
# Prepare lists from config files
load_whitelist()
load_blacklists()
load_predefined_lists()
# Start processing BLs in threads
threads = [threading.Thread(target=process_url, args=(url,)) for url in urls]
for t in threads:
t.start()
for t in threads:
t.join()
save_config_file()
print("Restarting unbound service")
subprocess.Popen(["pluginctl", "-s", "unbound", "restart"])
exit(0)

View File

@ -51,3 +51,10 @@ command:/usr/local/bin/flock -n -E 0 -o /tmp/unbound_start.lock /usr/local/opnse
parameters:
type:script
message:Start Unbound
[dnsbl]
command:/usr/local/opnsense/scripts/unbound/dnsbl.py
parameters:
type:script
message:fetching and applying DNSBLs
description: Download Unbound DNSBLs and restart

View File

@ -1,5 +1,9 @@
###################################################################
# Compatibility layer, to be removed in a future version
# - keeps old plugin cron jobs functional
###################################################################
[dnsbl]
command:/usr/local/opnsense/scripts/OPNsense/Unboundplus/dnsbl.py
command:/usr/local/opnsense/scripts/unbound/dnsbl.py
parameters:
type:script
message:fetching and applying DNSBLs

View File

@ -1 +1,6 @@
root.min.hints:/var/unbound/root.hints
dnsbl.inc:/var/unbound/etc/dnsbl.inc
whitelist.inc:/var/unbound/etc/whitelist.inc
miscellaneous.conf:/var/unbound/etc/miscellaneous.conf
dot.conf:/var/unbound/etc/dot.conf
lists.inc:/var/unbound/etc/lists.inc

View File

@ -1,5 +0,0 @@
dnsbl.inc:/var/unbound/etc/dnsbl.inc
whitelist.inc:/var/unbound/etc/whitelist.inc
miscellaneous.conf:/var/unbound/etc/miscellaneous.conf
dot.conf:/var/unbound/etc/dot.conf
lists.inc:/var/unbound/etc/lists.inc