lucaspalomodevelop/core
1
0
Fork 0
mirror of https://github.com/lucaspalomodevelop/core.git synced 2026-03-14 00:24:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

(firewall, aliases) add geoip

Browse Source
This commit is contained in:
Ad Schellevis 2016-04-11 16:42:48 +02:00
parent a75bf5613c
commit 98cfb7d46f
3 changed files with 215 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
src/etc/rc.update_urltables
View File

@ -10,16 +10,15 @@ if (!isset($config['aliases']['alias'])) {
exit;
}
// Gather list of urltable aliases
// Gather list of urltable / geoip aliases
$todo = array();
$download_geoip = false;
foreach ($config['aliases']['alias'] as $alias) {
if (preg_match('/urltable/i', $alias['type'])) {
$tmp = array();
$tmp['type'] = $alias['type'];
$tmp['name'] = $alias['name'];
$tmp['url'] = $alias['url'];
$tmp['freq'] = $alias['updatefreq'];
$todo[] = $tmp;
$todo[] = $alias;
} elseif ($alias['type'] == 'geoip') {
$todo[] = $alias;
$download_geoip = true;
}
}
@ -33,24 +32,49 @@ if (count($todo) > 0) {
sleep($wait);
}
// download geoip database
if ($download_geoip) {
// download the geoip database, first check if we haven't already done so the last day
if (!is_file('/usr/local/share/GeoIP/alias/NL-IPv4') || (time() - filemtime('/usr/local/share/GeoIP/alias/NL-IPv4')) > (86400 - 90)) {
log_error("{$argv[0]}: Download GeoIP database");
exec('/usr/local/opnsense/scripts/filter/download_geoip.py');
} else {
log_error("{$argv[0]}: GeoIP database doesn't need updating");
}
}
log_error("{$argv[0]}: Starting URL table alias updates");
$filter_reload = false;
foreach ($todo as $t) {
$r = process_alias_urltable($t['name'], $t['url'], $t['freq']);
if ($r == 1) {
$result = "";
// TODO: Change it when pf supports tables with ports
if ($t['type'] == "urltable") {
exec("/sbin/pfctl -t " . escapeshellarg($t['name']) . " -T replace -f /var/db/aliastables/" . escapeshellarg($t['name']) . ".txt 2>&1", $result);
foreach ($todo as $alias) {
if (preg_match('/urltable/i', $alias['type'])) {
$r = process_alias_urltable($alias['name'], $alias['url'], $alias['updatefreq']);
if ($r == 1) {
if ($alias['type'] == "urltable") {
exec("/sbin/pfctl -t " . escapeshellarg($alias['name']) . " -T replace -f /var/db/aliastables/" . escapeshellarg($alias['name']) . ".txt 2>&1", $result);
log_error("{$argv[0]}: Updated {$alias['name']} content from {$alias['url']}: ". $result[count($result)-1]);
} else {
$filter_reload = true;
}
} elseif ($r == -1) {
log_error("{$argv[0]}: {$alias['name']} does not need updating.");
} else {
$filter_reload = true;
log_error("{$argv[0]}: ERROR: could not update {$alias['name']} content from {$alias['url']}");
}
log_error("{$argv[0]}: Updated {$t['name']} content from {$t['url']}: {$result[0]}");
} elseif ($r == -1) {
log_error("{$argv[0]}: {$t['name']} does not need updating.");
} else {
log_error("{$argv[0]}: ERROR: could not update {$t['name']} content from {$t['url']}");
} elseif ($alias['type'] == 'geoip') {
// concat geoip countries and load into pf table
$alias_content = "";
foreach (explode(' ', $alias['address']) as $country_code) {
if (strlen($country_code) == 2 && in_array($alias['proto'], array('IPv4', 'IPv6'))) {
$filename = "/usr/local/share/GeoIP/alias/".$country_code."-".$alias['proto'];
if (is_file($filename)) {
$alias_content .= file_get_contents($filename);
}
}
}
file_put_contents('/var/db/aliastables/'.basename($alias['name']).'.txt', $alias_content);
exec("/sbin/pfctl -t " . escapeshellarg($alias['name']) . " -T replace -f /var/db/aliastables/" . escapeshellarg($alias['name']) . ".txt 2>&1", $result);
log_error("{$argv[0]}: Updated {$alias['name']} content from geoip database: ". $result[count($result)-1]);
}
}

87
src/opnsense/scripts/filter/download_geoip.py Executable file
View File

@ -0,0 +1,87 @@
#!/usr/local/bin/python2.7
"""
Copyright (c) 2016 Ad Schellevis
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
--------------------------------------------------------------------------------------
download maxmind GeoLite2 Free database into easy to use alias files [<COUNTRY>-<PROTO>] located
in /usr/local/share/GeoIP/alias
"""
import tempfile
import subprocess
import os
import sys
import ujson
import requests
import zipfile
# define geoip download location
url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip'
if __name__ == '__main__':
address_count = 0
file_count = 0
# flush data from remote url to temp file and unpack from there
with tempfile.NamedTemporaryFile() as tmp_stream:
r = requests.get(url)
if r.status_code == 200:
tmp_stream.write(r.content)
tmp_stream.seek(0)
with zipfile.ZipFile(tmp_stream, mode='r', compression=zipfile.ZIP_DEFLATED) as zf:
# fetch zip file contents
file_handles = dict()
for item in zf.infolist():
if item.file_size > 0:
file_handles[os.path.basename(item.filename)] = item
# only process geo ip data when archive contains country definitions
if 'GeoLite2-Country-Locations-en.csv' in file_handles:
country_codes = dict()
# parse geoname_id to country code map
for line in zf.open(file_handles['GeoLite2-Country-Locations-en.csv']).read().split('\n'):
parts = line.split(',')
if len(parts) > 4 and len(parts[4]) >= 1 and len(parts[4]) <= 3:
country_codes[parts[0]] = parts[4]
# process all details into files per country / protocol
for proto in ['IPv4', 'IPv6']:
if 'GeoLite2-Country-Blocks-%s.csv' % proto in file_handles:
output_handles = dict()
for line in zf.open(file_handles['GeoLite2-Country-Blocks-%s.csv' % proto]).read().split('\n'):
parts = line.split(',')
if len(parts) > 3 and parts[1] in country_codes:
country_code = country_codes[parts[1]]
if country_code not in output_handles:
if not os.path.exists('/usr/local/share/GeoIP/alias'):
os.makedirs('/usr/local/share/GeoIP/alias')
output_handles[country_code] = open(
'/usr/local/share/GeoIP/alias/%s-%s'%(country_code,proto), 'w'
)
file_count += 1
output_handles[country_code].write("%s\n" % parts[0])
address_count += 1
for country_code in output_handles:
output_handles[country_code].close()
# output files and lines processed
print ('%d files written, with a total number of %d lines' % (file_count, address_count))

96
src/www/firewall_aliases_edit.php
View File

@ -33,6 +33,23 @@
require_once("guiconfig.inc");
require_once("pfsense-utils.inc");
/**
* generate simple country selection list for geoip
*/
function geoip_countries()
{
$result = array();
foreach (explode("\n", file_get_contents('/usr/local/opnsense/contrib/tzdata/iso3166.tab')) as $line) {
$line = trim($line);
if (strlen($line) > 3 && substr($line, 0, 1) != '#') {
$code = substr($line, 0, 2);
$name = trim(substr($line, 2, 9999));
$result[$code] = $name;
}
}
return $result;
}
if (!isset($config['aliases']) || !is_array($config['aliases'])) {
$config['aliases'] = array();
}
@ -45,7 +62,7 @@ $pconfig = array();
if ($_SERVER['REQUEST_METHOD'] === 'GET') {
if (isset($_GET['id']) && is_numericint($_GET['id']) && isset($a_aliases[$_GET['id']])) {
$id = $_GET['id'];
foreach (array("name", "detail", "address", "type", "descr", "updatefreq", "aliasurl", "url") as $fieldname) {
foreach (array("name", "detail", "address", "type", "descr", "updatefreq", "aliasurl", "url", "proto") as $fieldname) {
if (isset($a_aliases[$id][$fieldname])) {
$pconfig[$fieldname] = $a_aliases[$id][$fieldname];
} else {
@ -65,7 +82,7 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') {
}
}
// initialize form fields, when not found present empty form
foreach (array("name", "detail", "address", "type", "descr", "updatefreq", "aliasurl", "url") as $fieldname) {
foreach (array("name", "detail", "address", "type", "descr", "updatefreq", "aliasurl", "url", "proto") as $fieldname) {
if (isset($id) && isset($a_aliases[$id][$fieldname])) {
$pconfig[$fieldname] = $a_aliases[$id][$fieldname];
} else {
@ -74,7 +91,7 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') {
}
} else {
// init empty
$init_fields = array("name", "detail", "address", "type", "descr", "updatefreq", "url");
$init_fields = array("name", "detail", "address", "type", "descr", "updatefreq", "url", "proto");
foreach ($init_fields as $fieldname) {
$pconfig[$fieldname] = null;
}
@ -92,7 +109,7 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') {
} elseif (strpos($pconfig['type'],'url') !== false) {
$pconfig['aliasurl'] = $pconfig['host_url'];
} else {
$pconfig['address'] = implode(' ',$pconfig['host_url']);
$pconfig['address'] = implode(' ', $pconfig['host_url']);
}
foreach ($pconfig['detail'] as &$detailDescr) {
@ -108,6 +125,7 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') {
if (isset($pconfig['submit'])) {
$input_errors = array();
// validate data
$country_codes = array_keys(geoip_countries());
foreach ($pconfig['host_url'] as $detail_entry) {
if ($pconfig['type'] == 'host') {
if (!is_domain($detail_entry) && !is_ipaddr($detail_entry)) {
@ -117,8 +135,11 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') {
if (!is_port($detail_entry) && !is_portrange($detail_entry)) {
$input_errors[] = sprintf(gettext("%s doesn't appear to be a valid port number"), $detail_entry) ;
}
} elseif ($pconfig['type'] == 'geoip') {
if (!in_array($detail_entry, $country_codes)) {
$input_errors[] = sprintf(gettext("%s doesn't appear to be a valid country code"), $detail_entry) ;
}
}
}
/* Check for reserved keyword names */
@ -126,15 +147,19 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') {
$reserved_keywords = array("all", "pass", "block", "out", "queue", "max", "min", "pptp", "pppoe", "L2TP", "OpenVPN", "IPsec");
// Add all Load balance names to reserved_keywords
if (is_array($config['load_balancer']['lbpool']))
foreach ($config['load_balancer']['lbpool'] as $lbpool)
if (is_array($config['load_balancer']['lbpool'])) {
foreach ($config['load_balancer']['lbpool'] as $lbpool) {
$reserved_keywords[] = $lbpool['name'];
}
}
$reserved_ifs = get_configured_interface_list(false, true);
$reserved_keywords = array_merge($reserved_keywords, $reserved_ifs, $reserved_table_names);
foreach ($reserved_keywords as $rk)
if ($rk == $pconfig['name'])
foreach ($reserved_keywords as $rk) {
if ($rk == $pconfig['name']) {
$input_errors[] = sprintf(gettext("Cannot use a reserved keyword as alias name %s"), $rk);
}
}
/* check for name interface description conflicts */
foreach ($config['interfaces'] as $interface) {
@ -174,7 +199,7 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') {
if (count($input_errors) == 0) {
// save to config
$copy_fields = array("name","detail","address","type","descr","updatefreq","aliasurl","url");
$copy_fields = array("name", "detail", "address", "type", "descr", "updatefreq", "aliasurl", "url");
$confItem = array();
foreach ($copy_fields as $fieldname) {
if (!empty($pconfig[$fieldname])) {
@ -182,6 +207,11 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') {
}
}
// proto is only for geoip selection
if ($pconfig['type'] == 'geoip') {
$confItem['proto'] = $pconfig['proto'];
}
/* Check to see if alias name needs to be
* renamed on referenced rules and such
*/
@ -271,6 +301,10 @@ include("head.inc");
$(".act-removerow").click(removeRow);
// link typeahead to new item
$(".fld_detail").typeahead({ source: document.all_aliases[$("#typeSelect").val()] });
// link geoip list to new item
$(".geoip_list").change(function(){
$(this).parent().find('input').val($(this).val());
});
});
$(".act-removerow").click(removeRow);
@ -288,6 +322,10 @@ include("head.inc");
$("#addNew").removeClass('hidden');
$('.act-removerow').removeClass('hidden');
}
$("#proto").addClass("hidden");
$(".geoip_list").addClass("hidden");
$(".host_url").removeClass("hidden");
$(".geoip_list > option").remove();
switch($("#typeSelect").val()) {
case 'urltable':
$("#detailsHeading1").html("<?=gettext("URL");?>");
@ -310,13 +348,27 @@ include("head.inc");
case 'port':
$("#detailsHeading1").html("<?=gettext("Port(s)");?>");
break;
case 'geoip':
$("#proto").removeClass("hidden");
$(".geoip_list").removeClass("hidden");
$(".host_url").addClass("hidden");
$("#detailsHeading1").html("<?=gettext("Country");?>");
$("#countries > option").clone().appendTo('.geoip_list');
$('.geoip_list').each(function(){
var url_item = $(this).parent().find('input').val();
$(this).val(url_item);
});
$('.geoip_list').change(function(){
$(this).parent().find('input').val($(this).val());
});
break;
}
$(".fld_detail").typeahead("destroy");
$(".fld_detail").typeahead({ source: document.all_aliases[$("#typeSelect").val()] });
}
$("#typeSelect").change(function(){
toggleType();
toggleType();
});
// collect all known aliases per type
@ -343,6 +395,16 @@ include("head.inc");
endforeach;
endif;
?>
</select>
<!-- push all available countries in a hidden select box for geoip -->
<select class="hidden" id="countries">
<?php
foreach (geoip_countries() as $code => $name):?>
<option value="<?=$code;?>"><?=$name;?></option>
<?php
endforeach;
?>
</select>
<section class="page-content-main">
@ -393,7 +455,15 @@ include("head.inc");
<option value="url_ports" <?=$pconfig['type'] == "url_ports" ? "selected=\"selected\"" : ""; ?>><?=gettext("URL (Ports)");?></option>
<option value="urltable" <?=$pconfig['type'] == "urltable" ? "selected=\"selected\"" : ""; ?>><?=gettext("URL Table (IPs)"); ?></option>
<option value="urltable_ports" <?=$pconfig['type'] == "urltable_ports" ? "selected=\"selected\"" : ""; ?>><?=gettext("URL Table (Ports)"); ?></option>
<option value="geoip" <?=$pconfig['type'] == "geoip" ? "selected=\"selected\"" : ""; ?>><?=gettext("GeoIP"); ?></option>
</select>
<div id="proto" class="hidden">
<small><?=gettext("Protocol");?></small><br/>
<select name="proto">
<option value="IPv4" <?=$pconfig['proto'] == "IPv4" ? "selected=\"selected\"" : ""; ?>><?=gettext("IPv4");?></option>
<option value="IPv6" <?=$pconfig['proto'] == "IPv6" ? "selected=\"selected\"" : ""; ?>><?=gettext("IPv6");?></option>
</select>
</div>
<div class="hidden" for="help_for_type">
<span class="text-info">
<?=gettext("Networks")?><br/>
@ -457,7 +527,9 @@ include("head.inc");
<div style="cursor:pointer;" class="act-removerow btn btn-default btn-xs" alt="remove"><span class="glyphicon glyphicon-minus"></span></div>
</td>
<td>
<input type="text" class="form-control" name="host_url[]" value="<?=$aliasurl;?>"/>
<select class="geoip_list hidden">
</select>
<input type="text" class="host_url" name="host_url[]" value="<?=$aliasurl;?>"/>
</td>
<td>
<input type="text" class="form-control" name="detail[]" value="<?= isset($detail_desc[$aliasid])?$detail_desc[$aliasid]:"";?>">