lucaspalomodevelop/core
1
0
Fork 0
mirror of https://github.com/lucaspalomodevelop/core.git synced 2026-03-15 00:54:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

(proxy, acl download) minor performance improvments + python3 compatibility additions

Browse Source
This commit is contained in:
Ad Schellevis 2016-12-13 10:29:20 +01:00
parent c5fe0e4c4f
commit 1dbaaca2b6
1 changed files with 36 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
src/opnsense/scripts/proxy/fetchACLs.py
View File

@ -29,6 +29,7 @@
import tempfile
import os
import sys
import json
import glob
import os.path
@ -36,9 +37,16 @@ import tarfile
import gzip
import zipfile
import syslog
import shutil
import urllib2
from ConfigParser import ConfigParser
if sys.version_info < (3, 0):
from ConfigParser import ConfigParser
from urllib2 import urlopen
from urllib2 import URLError
from urllib2 import HTTPError
else:
from configparser import ConfigParser
from urllib.request import urlopen
from urllib.error import URLError
from urllib.error import HTTPError
import requests
acl_config_fn = '/usr/local/etc/squid/externalACLs.conf'
@ -79,8 +87,13 @@ class Downloader(object):
req = requests.get(**req_opts)
if req.status_code == 200:
self._source_handle = tempfile.NamedTemporaryFile()
shutil.copyfileobj(req.raw, self._source_handle)
self._source_handle = tempfile.NamedTemporaryFile('wb+', 10240)
while True:
data = req.raw.read(10240)
if not data:
break
else:
self._source_handle.write(data)
self._source_handle.seek(0)
else:
syslog.syslog(syslog.LOG_ERR, 'proxy acl: error downloading %s (http code: %s)' % (self._url,
@ -88,17 +101,17 @@ class Downloader(object):
elif self._url.lower().startswith('ftp://'):
# FTP download
try:
f = urllib2.urlopen(self._url, timeout=self._timeout)
self._source_handle = tempfile.NamedTemporaryFile()
f = urlopen(self._url, timeout=self._timeout)
self._source_handle = tempfile.NamedTemporaryFile('wb+', 10240)
while True:
data = f.read(1024)
data = f.read(10240)
if not data:
break
else:
self._source_handle.write(data)
self._source_handle.seek(0)
f.close()
except (urllib2.URLError, urllib2.HTTPError, IOError) as e:
except (URLError, HTTPError, IOError) as e:
syslog.syslog(syslog.LOG_ERR, 'proxy acl: error downloading %s' % self._url)
else:
syslog.syslog(syslog.LOG_ERR, 'proxy acl: unsupported protocol for %s' % self._url)
@ -141,12 +154,16 @@ class Downloader(object):
""" download / unpack ACL
:return: iterator filename, type, content
"""
is_python3 = sys.version_info >= (3, 0)
self.fetch()
for filename, filehandle in self.get_files():
basefilename = os.path.basename(filename).lower()
file_ext = filename.split('.')[-1].lower()
while True:
line = filehandle.readline()
if is_python3:
line = filehandle.readline().decode(encoding='utf-8', errors='ignore')
else:
line = filehandle.readline()
if not line:
break
yield filename, basefilename, file_ext, line
@ -180,7 +197,7 @@ class DomainSorter(object):
target = chr(i + 1)
setid = int(i / (sets / self._num_targets))
if setid not in self._buckets:
self._buckets[setid] = tempfile.NamedTemporaryFile()
self._buckets[setid] = tempfile.NamedTemporaryFile('w+', 10240)
self._sort_map[target] = self._buckets[setid]
def write(self, data):
@ -203,7 +220,8 @@ class DomainSorter(object):
"""
target = key[0]
if target in self._sort_map:
self._sort_map[target].write('%s%s%s\n' % (key, self._seperator, value))
for part in (key, self._seperator, value, '\n'):
self._sort_map[target].write(part)
else:
# not supposed to happen, every key should have a calculated target pool
pass
@ -245,7 +263,7 @@ class DomainSorter(object):
"""
if self._target_filename is not None and self._target_mode is not None:
# flush to file on close
with open(self._target_filename, self._target_mode) as f_out:
with open(self._target_filename, self._target_mode, buffering=10240) as f_out:
prev_line = None
for line in self.reader():
line = line.lstrip('.')
@ -338,11 +356,12 @@ def main():
continue
if filetype in targets and targets[filetype]['handle'] is None:
targets[filetype]['handle'] = targets[filetype]['class'](targets[filetype]['filename'],'wb')
targets[filetype]['handle'] = targets[filetype]['class'](targets[filetype]['filename'],'w')
if filetype in targets:
targets[filetype]['handle'].write('%s\n' % line)
targets[filetype]['handle'].write(line)
targets[filetype]['handle'].write('\n')
# save index to disc
with open('%s.index' % target_filename, 'wb') as idx_out:
with open('%s.index' % target_filename, 'w', buffering=10240) as idx_out:
index_data = dict()
for filename in all_filenames:
if len(filename.split('/')) > 2:
@ -361,7 +380,7 @@ def main():
os.remove(targets[filetype]['filename'])
elif not os.path.isfile(targets[filetype]['filename']):
# no data fetched and no file available, create new empty file
with open(targets[filetype]['filename'], 'wb') as target_out:
with open(targets[filetype]['filename'], 'w') as target_out:
target_out.write("")