mirror of
https://github.com/lucaspalomodevelop/core.git
synced 2026-03-15 09:04:39 +00:00
GeoIP download settings, work in progress for https://github.com/opnsense/core/issues/3856
This commit is contained in:
Ad Schellevis
parent
bb35688f0e
commit
789eac7d99
@ -39,6 +39,7 @@ class AliasController extends IndexController
|
||||
{
|
||||
// include dialog form definitions
|
||||
$this->view->selected_alias = $selected;
|
||||
$this->view->formGeoIPSettings = $this->getForm("geoIPSettings");
|
||||
$this->view->pick('OPNsense/Firewall/alias');
|
||||
}
|
||||
}
|
||||
|
||||
@ -313,4 +313,26 @@ class AliasController extends ApiMutableModelControllerBase
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* get geoip settings (and stats)
|
||||
*/
|
||||
public function getGeoIPAction()
|
||||
{
|
||||
$result = array();
|
||||
if ($this->request->isGet()) {
|
||||
$result[static::$internalModelName] = array();
|
||||
$node = $this->getModel()->getNodeByReference('geoip');
|
||||
if ($node != null) {
|
||||
$result[static::$internalModelName]['geoip'] = $node->getNodes();
|
||||
}
|
||||
if (file_exists('/usr/local/share/GeoIP/alias.stats')) {
|
||||
$stats = json_decode(file_get_contents('/usr/local/share/GeoIP/alias.stats'), true);
|
||||
$result[static::$internalModelName]['geoip'] = array_merge(
|
||||
$result[static::$internalModelName]['geoip'], $stats
|
||||
);
|
||||
}
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,20 @@
|
||||
<form>
|
||||
<field>
|
||||
<id>alias.geoip.timestamp</id>
|
||||
<label>Last updated</label>
|
||||
<type>info</type>
|
||||
<help><![CDATA[Last updated timestamp.]]></help>
|
||||
</field>
|
||||
<field>
|
||||
<id>alias.geoip.address_count</id>
|
||||
<label>Total number of ranges</label>
|
||||
<type>info</type>
|
||||
<help><![CDATA[Total number of entries in downloaded set.]]></help>
|
||||
</field>
|
||||
<field>
|
||||
<id>alias.geoip.url</id>
|
||||
<label>Url</label>
|
||||
<type>text</type>
|
||||
<help><![CDATA[Location to fetch geoip address ranges from.]]></help>
|
||||
</field>
|
||||
</form>
|
||||
@ -5,6 +5,10 @@
|
||||
Firewall aliases
|
||||
</description>
|
||||
<items>
|
||||
<geoip>
|
||||
<url type="UrlField">
|
||||
</url>
|
||||
</geoip>
|
||||
<aliases>
|
||||
<alias type="ArrayField">
|
||||
<enabled type="BooleanField">
|
||||
|
||||
@ -330,22 +330,46 @@
|
||||
});
|
||||
});
|
||||
|
||||
let data_get_map = {'frm_GeopIPSettings':"/api/firewall/alias/getGeoIP"};
|
||||
mapDataToFormUI(data_get_map).done(function(data){
|
||||
formatTokenizersUI();
|
||||
$('.selectpicker').selectpicker('refresh');
|
||||
});
|
||||
|
||||
/**
|
||||
* reconfigure
|
||||
*/
|
||||
$("#reconfigureAct").click(function(){
|
||||
$("#reconfigureAct_progress").addClass("fa fa-spinner fa-pulse");
|
||||
ajaxCall("/api/firewall/alias/reconfigure", {}, function(data,status) {
|
||||
// when done, disable progress animation.
|
||||
$("#reconfigureAct_progress").removeClass("fa fa-spinner fa-pulse");
|
||||
saveFormToEndpoint("/api/firewall/alias/set", 'frm_GeopIPSettings', function(){
|
||||
ajaxCall("/api/firewall/alias/reconfigure", {}, function(data,status) {
|
||||
// when done, disable progress animation.
|
||||
$("#reconfigureAct_progress").removeClass("fa fa-spinner fa-pulse");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// update history on tab state and implement navigation
|
||||
if (window.location.hash != "") {
|
||||
$('a[href="' + window.location.hash + '"]').click();
|
||||
} else {
|
||||
$('a[href="#aliases"]').click();
|
||||
}
|
||||
|
||||
$('.nav-tabs a').on('shown.bs.tab', function (e) {
|
||||
history.pushState(null, null, e.target.hash);
|
||||
});
|
||||
|
||||
});
|
||||
</script>
|
||||
|
||||
<section class="page-content-main">
|
||||
<div class="container-fluid">
|
||||
<ul class="nav nav-tabs" data-tabs="tabs" id="maintabs">
|
||||
<li><a data-toggle="tab" href="#aliases" id="aliases_tab">{{ lang._('Aliases') }}</a></li>
|
||||
<li><a data-toggle="tab" href="#geoip" id="geoip_tab">{{ lang._('GeopIP settings') }}</a></li>
|
||||
</ul>
|
||||
|
||||
<div class="tab-content content-box">
|
||||
<div id="aliases" class="tab-pane fade in">
|
||||
<div class="row">
|
||||
<section class="col-xs-12">
|
||||
<div class="content-box">
|
||||
@ -380,21 +404,27 @@
|
||||
</tr>
|
||||
</tfoot>
|
||||
</table>
|
||||
<div class="col-md-12">
|
||||
<div id="aliasChangeMessage" class="alert alert-info" style="display: none" role="alert">
|
||||
{{ lang._('After changing settings, please remember to apply them with the button below') }}
|
||||
</div>
|
||||
<hr/>
|
||||
<button class="btn btn-primary" id="reconfigureAct" type="button"><b>{{ lang._('Apply') }}</b> <i id="reconfigureAct_progress" class=""></i></button>
|
||||
<br/><br/>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
</div>
|
||||
<div id="geoip" class="tab-pane fade in">
|
||||
{{ partial("layout_partials/base_form",['fields':formGeoIPSettings,'id':'frm_GeopIPSettings'])}}
|
||||
</div>
|
||||
</div>
|
||||
<section class="page-content-main">
|
||||
<div class="content-box">
|
||||
<div class="col-md-12">
|
||||
<br/>
|
||||
<div id="aliasChangeMessage" class="alert alert-info" style="display: none" role="alert">
|
||||
{{ lang._('After changing settings, please remember to apply them with the button below') }}
|
||||
</div>
|
||||
<button class="btn btn-primary" id="reconfigureAct" type="button"><b>{{ lang._('Apply') }}</b> <i id="reconfigureAct_progress" class=""></i></button>
|
||||
<br/><br/>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
{# Edit dialog #}
|
||||
<div class="modal fade" id="DialogAlias" tabindex="-1" role="dialog" aria-labelledby="DialogAliasLabel" aria-hidden="true">
|
||||
<div class="modal-backdrop fade in"></div>
|
||||
|
||||
@ -32,4 +32,4 @@
|
||||
from lib.geoip import download_geolite
|
||||
|
||||
# output files and lines processed
|
||||
print ('%d files written, with a total number of %d lines' % download_geolite())
|
||||
print ('%(file_count)d files written, with a total number of %(address_count)d lines' % download_geolite())
|
||||
|
||||
@ -172,7 +172,7 @@ class Alias(object):
|
||||
if (time.time() - fstat.st_mtime) < (86400 - 90):
|
||||
do_update = False
|
||||
if do_update:
|
||||
syslog.syslog(syslog.LOG_ERR, 'geoip updated (files: %s lines: %s)' % geoip.download_geolite())
|
||||
syslog.syslog(syslog.LOG_ERR, 'geoip updated (files: %(file_count)d lines: %(address_count)d)' % geoip.download_geolite())
|
||||
|
||||
for proto in self._proto.split(','):
|
||||
geoip_filename = "/usr/local/share/GeoIP/alias/%s-%s" % (geoitem, proto)
|
||||
|
||||
@ -27,6 +27,7 @@
|
||||
download maxmind GeoLite2 Free database into easy to use alias files [<COUNTRY>-<PROTO>] located
|
||||
in /usr/local/share/GeoIP/alias
|
||||
"""
|
||||
import datetime
|
||||
import tempfile
|
||||
import subprocess
|
||||
import os
|
||||
@ -34,52 +35,65 @@ import sys
|
||||
import ujson
|
||||
import requests
|
||||
import zipfile
|
||||
from configparser import ConfigParser
|
||||
|
||||
|
||||
def download_geolite():
|
||||
# define geoip download location
|
||||
url = 'https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip'
|
||||
updater_conf='/usr/local/etc/filter_geoip.conf'
|
||||
stats_output = '/usr/local/share/GeoIP/alias.stats'
|
||||
url = None
|
||||
if os.path.exists(updater_conf):
|
||||
cnf = ConfigParser()
|
||||
cnf.read(updater_conf)
|
||||
if cnf.has_section('settings') and cnf.has_option('settings', 'url'):
|
||||
url = cnf.get('settings', 'url').strip()
|
||||
|
||||
address_count = 0
|
||||
file_count = 0
|
||||
# flush data from remote url to temp file and unpack from there
|
||||
with tempfile.NamedTemporaryFile() as tmp_stream:
|
||||
r = requests.get(url)
|
||||
if r.status_code == 200:
|
||||
tmp_stream.write(r.content)
|
||||
tmp_stream.seek(0)
|
||||
with zipfile.ZipFile(tmp_stream, mode='r', compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
# fetch zip file contents
|
||||
file_handles = dict()
|
||||
for item in zf.infolist():
|
||||
if item.file_size > 0:
|
||||
file_handles[os.path.basename(item.filename)] = item
|
||||
# only process geo ip data when archive contains country definitions
|
||||
if 'GeoLite2-Country-Locations-en.csv' in file_handles:
|
||||
country_codes = dict()
|
||||
# parse geoname_id to country code map
|
||||
for line in zf.open(file_handles['GeoLite2-Country-Locations-en.csv']).read().decode().split('\n'):
|
||||
parts = line.split(',')
|
||||
if len(parts) > 4 and len(parts[4]) >= 1 and len(parts[4]) <= 3:
|
||||
country_codes[parts[0]] = parts[4]
|
||||
# process all details into files per country / protocol
|
||||
for proto in ['IPv4', 'IPv6']:
|
||||
if 'GeoLite2-Country-Blocks-%s.csv' % proto in file_handles:
|
||||
output_handles = dict()
|
||||
country_blocks = zf.open(file_handles['GeoLite2-Country-Blocks-%s.csv' % proto]).read()
|
||||
for line in country_blocks.decode().split('\n'):
|
||||
parts = line.split(',')
|
||||
if len(parts) > 3 and parts[1] in country_codes:
|
||||
country_code = country_codes[parts[1]]
|
||||
if country_code not in output_handles:
|
||||
if not os.path.exists('/usr/local/share/GeoIP/alias'):
|
||||
os.makedirs('/usr/local/share/GeoIP/alias')
|
||||
output_handles[country_code] = open(
|
||||
'/usr/local/share/GeoIP/alias/%s-%s'%(country_code,proto), 'w'
|
||||
)
|
||||
file_count += 1
|
||||
output_handles[country_code].write("%s\n" % parts[0])
|
||||
address_count += 1
|
||||
for country_code in output_handles:
|
||||
output_handles[country_code].close()
|
||||
result = {'address_count': 0 , 'file_count': 0, 'timestamp': None}
|
||||
if url is not None:
|
||||
# flush data from remote url to temp file and unpack from there
|
||||
with tempfile.NamedTemporaryFile() as tmp_stream:
|
||||
r = requests.get(url)
|
||||
if r.status_code == 200:
|
||||
tmp_stream.write(r.content)
|
||||
tmp_stream.seek(0)
|
||||
with zipfile.ZipFile(tmp_stream, mode='r', compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
# fetch zip file contents
|
||||
file_handles = dict()
|
||||
for item in zf.infolist():
|
||||
if item.file_size > 0:
|
||||
file_handles[os.path.basename(item.filename)] = item
|
||||
# only process geo ip data when archive contains country definitions
|
||||
if 'GeoLite2-Country-Locations-en.csv' in file_handles:
|
||||
dt = datetime.datetime(*file_handles['GeoLite2-Country-Locations-en.csv'].date_time).isoformat()
|
||||
result['timestamp'] = dt
|
||||
country_codes = dict()
|
||||
# parse geoname_id to country code map
|
||||
for line in zf.open(file_handles['GeoLite2-Country-Locations-en.csv']).read().decode().split('\n'):
|
||||
parts = line.split(',')
|
||||
if len(parts) > 4 and len(parts[4]) >= 1 and len(parts[4]) <= 3:
|
||||
country_codes[parts[0]] = parts[4]
|
||||
# process all details into files per country / protocol
|
||||
for proto in ['IPv4', 'IPv6']:
|
||||
if 'GeoLite2-Country-Blocks-%s.csv' % proto in file_handles:
|
||||
output_handles = dict()
|
||||
country_blocks = zf.open(file_handles['GeoLite2-Country-Blocks-%s.csv' % proto]).read()
|
||||
for line in country_blocks.decode().split('\n'):
|
||||
parts = line.split(',')
|
||||
if len(parts) > 3 and parts[1] in country_codes:
|
||||
country_code = country_codes[parts[1]]
|
||||
if country_code not in output_handles:
|
||||
if not os.path.exists('/usr/local/share/GeoIP/alias'):
|
||||
os.makedirs('/usr/local/share/GeoIP/alias')
|
||||
output_handles[country_code] = open(
|
||||
'/usr/local/share/GeoIP/alias/%s-%s'%(country_code,proto), 'w'
|
||||
)
|
||||
result['file_count'] += 1
|
||||
output_handles[country_code].write("%s\n" % parts[0])
|
||||
result['address_count'] += 1
|
||||
for country_code in output_handles:
|
||||
output_handles[country_code].close()
|
||||
|
||||
return (file_count, address_count)
|
||||
open(stats_output,'w').write(ujson.dumps(result))
|
||||
|
||||
return result
|
||||
|
||||
@ -1 +1,2 @@
|
||||
filter_tables.conf:/usr/local/etc/filter_tables.conf
|
||||
filter_geoip.conf:/usr/local/etc/filter_geoip.conf
|
||||
|
||||
@ -0,0 +1,6 @@
|
||||
[settings]
|
||||
{% if helpers.exists('OPNsense.Firewall.Alias.geoip.url') %}
|
||||
url={{OPNsense.Firewall.Alias.geoip.url}}
|
||||
{% else %}
|
||||
url=
|
||||
{% endif %}
|
||||
Loading…
x
Reference in New Issue
Block a user