mirror of
https://github.com/lucaspalomodevelop/core.git
synced 2026-03-14 08:34:39 +00:00
IDS/IPS rules, add support for version checks, closes https://github.com/opnsense/core/issues/2377
This commit is contained in:
Ad Schellevis
parent
64811322fb
commit
f43a5c8c58
@ -34,6 +34,9 @@ import gzip
|
||||
import zipfile
|
||||
import tempfile
|
||||
import requests
|
||||
import json
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
|
||||
class Downloader(object):
|
||||
@ -110,16 +113,14 @@ class Downloader(object):
|
||||
else:
|
||||
return src.read()
|
||||
|
||||
def download(self, proto, url, url_filename, filename, input_filter, auth = None, headers=None):
|
||||
""" download ruleset file
|
||||
:param proto: protocol (http,https)
|
||||
def fetch(self, url, auth=None, headers=None):
|
||||
""" Fetch file from remote location and save to temp, return filehandle pointed to start of temp file.
|
||||
Results are cached, which prevents downloading the same archive twice for example.
|
||||
:param url: download url
|
||||
:param filename: target filename
|
||||
:param input_filter: filter to use on received data before save
|
||||
:param auth: authentication
|
||||
:param headers: headers to send
|
||||
"""
|
||||
if proto in ('http', 'https'):
|
||||
if str(url).split(':')[0].lower() in ('http', 'https'):
|
||||
frm_url = url.replace('//', '/').replace(':/', '://')
|
||||
# stream to temp file
|
||||
if frm_url not in self._download_cache:
|
||||
@ -141,30 +142,81 @@ class Downloader(object):
|
||||
break
|
||||
else:
|
||||
src.write(data)
|
||||
src.seek(0)
|
||||
self._download_cache[frm_url] = src
|
||||
|
||||
# process rules from tempfile (prevent duplicate download for files within an archive)
|
||||
if frm_url in self._download_cache:
|
||||
try:
|
||||
target_filename = '%s/%s' % (self._target_dir, filename)
|
||||
save_data = self._unpack(self._download_cache[frm_url], url, url_filename)
|
||||
save_data = self.filter(save_data, input_filter)
|
||||
open(target_filename, 'w', buffering=10240).write(save_data)
|
||||
except IOError:
|
||||
syslog.syslog(syslog.LOG_ERR, 'cannot write to %s' % target_filename)
|
||||
return None
|
||||
syslog.syslog(syslog.LOG_INFO, 'download completed for %s' % frm_url)
|
||||
else:
|
||||
syslog.syslog(syslog.LOG_ERR, 'download failed for %s' % frm_url)
|
||||
if frm_url in self._download_cache:
|
||||
self._download_cache[frm_url].seek(0)
|
||||
return self._download_cache[frm_url]
|
||||
else:
|
||||
return None
|
||||
|
||||
def fetch_version_hash(self, check_url, input_filter, auth=None, headers=None):
|
||||
""" Calculate a hash value using the download settings and a predefined version url (check_url).
|
||||
:param check_url: download url, version identifier
|
||||
:param input_filter: filter to use on received data before save
|
||||
:param auth: authentication
|
||||
:param headers: headers to send
|
||||
:return: None or hash
|
||||
"""
|
||||
if check_url is not None:
|
||||
# when no check url provided, assume different
|
||||
if self.is_supported(check_url):
|
||||
version_handle = self.fetch(url=check_url, auth=auth, headers=headers)
|
||||
if version_handle:
|
||||
hash_value = [json.dumps(input_filter), json.dumps(auth),
|
||||
json.dumps(headers), version_handle.read()]
|
||||
return hashlib.md5('\n'.join(hash_value)).hexdigest()
|
||||
return None
|
||||
|
||||
def installed_file_hash(self, filename):
|
||||
""" Fetch file version hash from header
|
||||
:param filename: target filename
|
||||
:return: None or hash
|
||||
"""
|
||||
target_filename = '%s/%s' % (self._target_dir, filename)
|
||||
if os.path.isfile(target_filename):
|
||||
with open(target_filename, 'r') as f_in:
|
||||
line = f_in.readline()
|
||||
if line.find("#@opnsense_download_hash:") == 0:
|
||||
return line.split(':')[1].strip()
|
||||
return None
|
||||
|
||||
def download(self, url, url_filename, filename, input_filter, auth=None, headers=None, version=None):
|
||||
""" download ruleset file
|
||||
:param url: download url
|
||||
:param url_filename: if provided the filename within the (packet) resource
|
||||
:param filename: target filename
|
||||
:param input_filter: filter to use on received data before save
|
||||
:param auth: authentication
|
||||
:param headers: headers to send
|
||||
:param version: version hash
|
||||
"""
|
||||
frm_url = url.replace('//', '/').replace(':/', '://')
|
||||
file_stream = self.fetch(url=url, auth=auth, headers=headers)
|
||||
if file_stream is not None:
|
||||
try:
|
||||
target_filename = '%s/%s' % (self._target_dir, filename)
|
||||
if version:
|
||||
save_data = "#@opnsense_download_hash:%s\n" % version
|
||||
else:
|
||||
save_data = ""
|
||||
save_data += self._unpack(file_stream, url, url_filename)
|
||||
save_data = self.filter(save_data, input_filter)
|
||||
open(target_filename, 'w', buffering=10240).write(save_data)
|
||||
except IOError:
|
||||
syslog.syslog(syslog.LOG_ERR, 'cannot write to %s' % target_filename)
|
||||
return None
|
||||
syslog.syslog(syslog.LOG_INFO, 'download completed for %s' % frm_url)
|
||||
else:
|
||||
syslog.syslog(syslog.LOG_ERR, 'download failed for %s' % frm_url)
|
||||
|
||||
@staticmethod
|
||||
def is_supported(proto):
|
||||
def is_supported(url):
|
||||
""" check if protocol is supported
|
||||
:param proto:
|
||||
:return:
|
||||
:param url: uri to request resource from
|
||||
:return: bool
|
||||
"""
|
||||
if proto in ['http', 'https']:
|
||||
if str(url).split(':')[0].lower() in ['http', 'https']:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
@ -108,7 +108,10 @@ class Metadata(object):
|
||||
else:
|
||||
metadata_record['url'] = ('%s/%s' % (metadata_record['source']['url'],
|
||||
metadata_record['filename']))
|
||||
|
||||
if rule_xml.find('version') is not None and 'url' in rule_xml.find('version').attrib:
|
||||
metadata_record['version_url'] = rule_xml.find('version').attrib['url']
|
||||
else:
|
||||
metadata_record['version_url'] = None
|
||||
if 'prefix' in src_location.attrib:
|
||||
description_prefix = "%s/" % src_location.attrib['prefix']
|
||||
else:
|
||||
|
||||
@ -27,11 +27,12 @@
|
||||
|
||||
--------------------------------------------------------------------------------------
|
||||
|
||||
update suricata rules
|
||||
update/download suricata rules
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import syslog
|
||||
import fcntl
|
||||
from ConfigParser import ConfigParser
|
||||
from lib import metadata
|
||||
@ -73,7 +74,7 @@ if __name__ == '__main__':
|
||||
for rule in md.list_rules(rule_properties):
|
||||
if 'url' in rule['source']:
|
||||
download_proto = str(rule['source']['url']).split(':')[0].lower()
|
||||
if dl.is_supported(download_proto):
|
||||
if dl.is_supported(url=rule['source']['url']):
|
||||
if rule['filename'] not in enabled_rulefiles:
|
||||
try:
|
||||
# remove configurable but unselected file
|
||||
@ -86,6 +87,13 @@ if __name__ == '__main__':
|
||||
auth = (rule['source']['username'], rule['source']['password'])
|
||||
else:
|
||||
auth = None
|
||||
dl.download(proto=download_proto, url=rule['url'], url_filename=rule['url_filename'],
|
||||
filename=rule['filename'], input_filter=input_filter, auth=auth,
|
||||
headers=rule['http_headers'])
|
||||
# when metadata supports versioning, check if either version or settings changed before download
|
||||
remote_hash = dl.fetch_version_hash(check_url=rule['version_url'], input_filter=input_filter,
|
||||
auth=auth, headers=rule['http_headers'])
|
||||
local_hash = dl.installed_file_hash(rule['filename'])
|
||||
if remote_hash is None or remote_hash != local_hash:
|
||||
dl.download(url=rule['url'], url_filename=rule['url_filename'],
|
||||
filename=rule['filename'], input_filter=input_filter, auth=auth,
|
||||
headers=rule['http_headers'], version=remote_hash)
|
||||
else:
|
||||
syslog.syslog(syslog.LOG_INFO, 'download skipped %s, same version' % rule['filename'])
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user