diff --git a/src/etc/rc.update_urltables b/src/etc/rc.update_urltables index 8f76ee77f..5a9fe20ec 100755 --- a/src/etc/rc.update_urltables +++ b/src/etc/rc.update_urltables @@ -10,16 +10,15 @@ if (!isset($config['aliases']['alias'])) { exit; } -// Gather list of urltable aliases +// Gather list of urltable / geoip aliases $todo = array(); +$download_geoip = false; foreach ($config['aliases']['alias'] as $alias) { if (preg_match('/urltable/i', $alias['type'])) { - $tmp = array(); - $tmp['type'] = $alias['type']; - $tmp['name'] = $alias['name']; - $tmp['url'] = $alias['url']; - $tmp['freq'] = $alias['updatefreq']; - $todo[] = $tmp; + $todo[] = $alias; + } elseif ($alias['type'] == 'geoip') { + $todo[] = $alias; + $download_geoip = true; } } @@ -33,24 +32,49 @@ if (count($todo) > 0) { sleep($wait); } + // download geoip database + if ($download_geoip) { + // download the geoip database, first check if we haven't already done so the last day + if (!is_file('/usr/local/share/GeoIP/alias/NL-IPv4') || (time() - filemtime('/usr/local/share/GeoIP/alias/NL-IPv4')) > (86400 - 90)) { + log_error("{$argv[0]}: Download GeoIP database"); + exec('/usr/local/opnsense/scripts/filter/download_geoip.py'); + } else { + log_error("{$argv[0]}: GeoIP database doesn't need updating"); + } + } + log_error("{$argv[0]}: Starting URL table alias updates"); $filter_reload = false; - foreach ($todo as $t) { - $r = process_alias_urltable($t['name'], $t['url'], $t['freq']); - if ($r == 1) { - $result = ""; - // TODO: Change it when pf supports tables with ports - if ($t['type'] == "urltable") { - exec("/sbin/pfctl -t " . escapeshellarg($t['name']) . " -T replace -f /var/db/aliastables/" . escapeshellarg($t['name']) . ".txt 2>&1", $result); + foreach ($todo as $alias) { + if (preg_match('/urltable/i', $alias['type'])) { + $r = process_alias_urltable($alias['name'], $alias['url'], $alias['updatefreq']); + if ($r == 1) { + if ($alias['type'] == "urltable") { + exec("/sbin/pfctl -t " . escapeshellarg($alias['name']) . " -T replace -f /var/db/aliastables/" . escapeshellarg($alias['name']) . ".txt 2>&1", $result); + log_error("{$argv[0]}: Updated {$alias['name']} content from {$alias['url']}: ". $result[count($result)-1]); + } else { + $filter_reload = true; + } + } elseif ($r == -1) { + log_error("{$argv[0]}: {$alias['name']} does not need updating."); } else { - $filter_reload = true; + log_error("{$argv[0]}: ERROR: could not update {$alias['name']} content from {$alias['url']}"); } - log_error("{$argv[0]}: Updated {$t['name']} content from {$t['url']}: {$result[0]}"); - } elseif ($r == -1) { - log_error("{$argv[0]}: {$t['name']} does not need updating."); - } else { - log_error("{$argv[0]}: ERROR: could not update {$t['name']} content from {$t['url']}"); + } elseif ($alias['type'] == 'geoip') { + // concat geoip countries and load into pf table + $alias_content = ""; + foreach (explode(' ', $alias['address']) as $country_code) { + if (strlen($country_code) == 2 && in_array($alias['proto'], array('IPv4', 'IPv6'))) { + $filename = "/usr/local/share/GeoIP/alias/".$country_code."-".$alias['proto']; + if (is_file($filename)) { + $alias_content .= file_get_contents($filename); + } + } + } + file_put_contents('/var/db/aliastables/'.basename($alias['name']).'.txt', $alias_content); + exec("/sbin/pfctl -t " . escapeshellarg($alias['name']) . " -T replace -f /var/db/aliastables/" . escapeshellarg($alias['name']) . ".txt 2>&1", $result); + log_error("{$argv[0]}: Updated {$alias['name']} content from geoip database: ". $result[count($result)-1]); } } diff --git a/src/opnsense/scripts/filter/download_geoip.py b/src/opnsense/scripts/filter/download_geoip.py new file mode 100755 index 000000000..683dd2589 --- /dev/null +++ b/src/opnsense/scripts/filter/download_geoip.py @@ -0,0 +1,87 @@ +#!/usr/local/bin/python2.7 + +""" + Copyright (c) 2016 Ad Schellevis + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------------- + download maxmind GeoLite2 Free database into easy to use alias files [-] located + in /usr/local/share/GeoIP/alias +""" +import tempfile +import subprocess +import os +import sys +import ujson +import requests +import zipfile + +# define geoip download location +url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip' + +if __name__ == '__main__': + address_count = 0 + file_count = 0 + # flush data from remote url to temp file and unpack from there + with tempfile.NamedTemporaryFile() as tmp_stream: + r = requests.get(url) + if r.status_code == 200: + tmp_stream.write(r.content) + tmp_stream.seek(0) + with zipfile.ZipFile(tmp_stream, mode='r', compression=zipfile.ZIP_DEFLATED) as zf: + # fetch zip file contents + file_handles = dict() + for item in zf.infolist(): + if item.file_size > 0: + file_handles[os.path.basename(item.filename)] = item + # only process geo ip data when archive contains country definitions + if 'GeoLite2-Country-Locations-en.csv' in file_handles: + country_codes = dict() + # parse geoname_id to country code map + for line in zf.open(file_handles['GeoLite2-Country-Locations-en.csv']).read().split('\n'): + parts = line.split(',') + if len(parts) > 4 and len(parts[4]) >= 1 and len(parts[4]) <= 3: + country_codes[parts[0]] = parts[4] + # process all details into files per country / protocol + for proto in ['IPv4', 'IPv6']: + if 'GeoLite2-Country-Blocks-%s.csv' % proto in file_handles: + output_handles = dict() + for line in zf.open(file_handles['GeoLite2-Country-Blocks-%s.csv' % proto]).read().split('\n'): + parts = line.split(',') + if len(parts) > 3 and parts[1] in country_codes: + country_code = country_codes[parts[1]] + if country_code not in output_handles: + if not os.path.exists('/usr/local/share/GeoIP/alias'): + os.makedirs('/usr/local/share/GeoIP/alias') + output_handles[country_code] = open( + '/usr/local/share/GeoIP/alias/%s-%s'%(country_code,proto), 'w' + ) + file_count += 1 + output_handles[country_code].write("%s\n" % parts[0]) + address_count += 1 + for country_code in output_handles: + output_handles[country_code].close() + + # output files and lines processed + print ('%d files written, with a total number of %d lines' % (file_count, address_count)) diff --git a/src/www/firewall_aliases_edit.php b/src/www/firewall_aliases_edit.php index 933edfe7d..ed1b0ba48 100644 --- a/src/www/firewall_aliases_edit.php +++ b/src/www/firewall_aliases_edit.php @@ -33,6 +33,23 @@ require_once("guiconfig.inc"); require_once("pfsense-utils.inc"); +/** + * generate simple country selection list for geoip + */ +function geoip_countries() +{ + $result = array(); + foreach (explode("\n", file_get_contents('/usr/local/opnsense/contrib/tzdata/iso3166.tab')) as $line) { + $line = trim($line); + if (strlen($line) > 3 && substr($line, 0, 1) != '#') { + $code = substr($line, 0, 2); + $name = trim(substr($line, 2, 9999)); + $result[$code] = $name; + } + } + return $result; +} + if (!isset($config['aliases']) || !is_array($config['aliases'])) { $config['aliases'] = array(); } @@ -45,7 +62,7 @@ $pconfig = array(); if ($_SERVER['REQUEST_METHOD'] === 'GET') { if (isset($_GET['id']) && is_numericint($_GET['id']) && isset($a_aliases[$_GET['id']])) { $id = $_GET['id']; - foreach (array("name", "detail", "address", "type", "descr", "updatefreq", "aliasurl", "url") as $fieldname) { + foreach (array("name", "detail", "address", "type", "descr", "updatefreq", "aliasurl", "url", "proto") as $fieldname) { if (isset($a_aliases[$id][$fieldname])) { $pconfig[$fieldname] = $a_aliases[$id][$fieldname]; } else { @@ -65,7 +82,7 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') { } } // initialize form fields, when not found present empty form - foreach (array("name", "detail", "address", "type", "descr", "updatefreq", "aliasurl", "url") as $fieldname) { + foreach (array("name", "detail", "address", "type", "descr", "updatefreq", "aliasurl", "url", "proto") as $fieldname) { if (isset($id) && isset($a_aliases[$id][$fieldname])) { $pconfig[$fieldname] = $a_aliases[$id][$fieldname]; } else { @@ -74,7 +91,7 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') { } } else { // init empty - $init_fields = array("name", "detail", "address", "type", "descr", "updatefreq", "url"); + $init_fields = array("name", "detail", "address", "type", "descr", "updatefreq", "url", "proto"); foreach ($init_fields as $fieldname) { $pconfig[$fieldname] = null; } @@ -92,7 +109,7 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') { } elseif (strpos($pconfig['type'],'url') !== false) { $pconfig['aliasurl'] = $pconfig['host_url']; } else { - $pconfig['address'] = implode(' ',$pconfig['host_url']); + $pconfig['address'] = implode(' ', $pconfig['host_url']); } foreach ($pconfig['detail'] as &$detailDescr) { @@ -108,6 +125,7 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') { if (isset($pconfig['submit'])) { $input_errors = array(); // validate data + $country_codes = array_keys(geoip_countries()); foreach ($pconfig['host_url'] as $detail_entry) { if ($pconfig['type'] == 'host') { if (!is_domain($detail_entry) && !is_ipaddr($detail_entry)) { @@ -117,8 +135,11 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') { if (!is_port($detail_entry) && !is_portrange($detail_entry)) { $input_errors[] = sprintf(gettext("%s doesn't appear to be a valid port number"), $detail_entry) ; } + } elseif ($pconfig['type'] == 'geoip') { + if (!in_array($detail_entry, $country_codes)) { + $input_errors[] = sprintf(gettext("%s doesn't appear to be a valid country code"), $detail_entry) ; + } } - } /* Check for reserved keyword names */ @@ -126,15 +147,19 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') { $reserved_keywords = array("all", "pass", "block", "out", "queue", "max", "min", "pptp", "pppoe", "L2TP", "OpenVPN", "IPsec"); // Add all Load balance names to reserved_keywords - if (is_array($config['load_balancer']['lbpool'])) - foreach ($config['load_balancer']['lbpool'] as $lbpool) + if (is_array($config['load_balancer']['lbpool'])) { + foreach ($config['load_balancer']['lbpool'] as $lbpool) { $reserved_keywords[] = $lbpool['name']; + } + } $reserved_ifs = get_configured_interface_list(false, true); $reserved_keywords = array_merge($reserved_keywords, $reserved_ifs, $reserved_table_names); - foreach ($reserved_keywords as $rk) - if ($rk == $pconfig['name']) + foreach ($reserved_keywords as $rk) { + if ($rk == $pconfig['name']) { $input_errors[] = sprintf(gettext("Cannot use a reserved keyword as alias name %s"), $rk); + } + } /* check for name interface description conflicts */ foreach ($config['interfaces'] as $interface) { @@ -174,7 +199,7 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') { if (count($input_errors) == 0) { // save to config - $copy_fields = array("name","detail","address","type","descr","updatefreq","aliasurl","url"); + $copy_fields = array("name", "detail", "address", "type", "descr", "updatefreq", "aliasurl", "url"); $confItem = array(); foreach ($copy_fields as $fieldname) { if (!empty($pconfig[$fieldname])) { @@ -182,6 +207,11 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') { } } + // proto is only for geoip selection + if ($pconfig['type'] == 'geoip') { + $confItem['proto'] = $pconfig['proto']; + } + /* Check to see if alias name needs to be * renamed on referenced rules and such */ @@ -271,6 +301,10 @@ include("head.inc"); $(".act-removerow").click(removeRow); // link typeahead to new item $(".fld_detail").typeahead({ source: document.all_aliases[$("#typeSelect").val()] }); + // link geoip list to new item + $(".geoip_list").change(function(){ + $(this).parent().find('input').val($(this).val()); + }); }); $(".act-removerow").click(removeRow); @@ -288,6 +322,10 @@ include("head.inc"); $("#addNew").removeClass('hidden'); $('.act-removerow').removeClass('hidden'); } + $("#proto").addClass("hidden"); + $(".geoip_list").addClass("hidden"); + $(".host_url").removeClass("hidden"); + $(".geoip_list > option").remove(); switch($("#typeSelect").val()) { case 'urltable': $("#detailsHeading1").html(""); @@ -310,13 +348,27 @@ include("head.inc"); case 'port': $("#detailsHeading1").html(""); break; + case 'geoip': + $("#proto").removeClass("hidden"); + $(".geoip_list").removeClass("hidden"); + $(".host_url").addClass("hidden"); + $("#detailsHeading1").html(""); + $("#countries > option").clone().appendTo('.geoip_list'); + $('.geoip_list').each(function(){ + var url_item = $(this).parent().find('input').val(); + $(this).val(url_item); + }); + $('.geoip_list').change(function(){ + $(this).parent().find('input').val($(this).val()); + }); + break; } $(".fld_detail").typeahead("destroy"); $(".fld_detail").typeahead({ source: document.all_aliases[$("#typeSelect").val()] }); } $("#typeSelect").change(function(){ - toggleType(); + toggleType(); }); // collect all known aliases per type @@ -343,6 +395,16 @@ include("head.inc"); endforeach; endif; ?> + + + +
@@ -393,7 +455,15 @@ include("head.inc"); + +