diff --git a/src/opnsense/scripts/suricata/lib/downloader.py b/src/opnsense/scripts/suricata/lib/downloader.py index 503c6cb38..a2f70fd5b 100644 --- a/src/opnsense/scripts/suricata/lib/downloader.py +++ b/src/opnsense/scripts/suricata/lib/downloader.py @@ -34,6 +34,9 @@ import gzip import zipfile import tempfile import requests +import json +import hashlib +import os class Downloader(object): @@ -110,16 +113,14 @@ class Downloader(object): else: return src.read() - def download(self, proto, url, url_filename, filename, input_filter, auth = None, headers=None): - """ download ruleset file - :param proto: protocol (http,https) + def fetch(self, url, auth=None, headers=None): + """ Fetch file from remote location and save to temp, return filehandle pointed to start of temp file. + Results are cached, which prevents downloading the same archive twice for example. :param url: download url - :param filename: target filename - :param input_filter: filter to use on received data before save :param auth: authentication :param headers: headers to send """ - if proto in ('http', 'https'): + if str(url).split(':')[0].lower() in ('http', 'https'): frm_url = url.replace('//', '/').replace(':/', '://') # stream to temp file if frm_url not in self._download_cache: @@ -141,30 +142,81 @@ class Downloader(object): break else: src.write(data) - src.seek(0) self._download_cache[frm_url] = src - # process rules from tempfile (prevent duplicate download for files within an archive) - if frm_url in self._download_cache: - try: - target_filename = '%s/%s' % (self._target_dir, filename) - save_data = self._unpack(self._download_cache[frm_url], url, url_filename) - save_data = self.filter(save_data, input_filter) - open(target_filename, 'w', buffering=10240).write(save_data) - except IOError: - syslog.syslog(syslog.LOG_ERR, 'cannot write to %s' % target_filename) - return None - syslog.syslog(syslog.LOG_INFO, 'download completed for %s' % frm_url) - else: - syslog.syslog(syslog.LOG_ERR, 'download failed for %s' % frm_url) + if frm_url in self._download_cache: + self._download_cache[frm_url].seek(0) + return self._download_cache[frm_url] + else: + return None + + def fetch_version_hash(self, check_url, input_filter, auth=None, headers=None): + """ Calculate a hash value using the download settings and a predefined version url (check_url). + :param check_url: download url, version identifier + :param input_filter: filter to use on received data before save + :param auth: authentication + :param headers: headers to send + :return: None or hash + """ + if check_url is not None: + # when no check url provided, assume different + if self.is_supported(check_url): + version_handle = self.fetch(url=check_url, auth=auth, headers=headers) + if version_handle: + hash_value = [json.dumps(input_filter), json.dumps(auth), + json.dumps(headers), version_handle.read()] + return hashlib.md5('\n'.join(hash_value)).hexdigest() + return None + + def installed_file_hash(self, filename): + """ Fetch file version hash from header + :param filename: target filename + :return: None or hash + """ + target_filename = '%s/%s' % (self._target_dir, filename) + if os.path.isfile(target_filename): + with open(target_filename, 'r') as f_in: + line = f_in.readline() + if line.find("#@opnsense_download_hash:") == 0: + return line.split(':')[1].strip() + return None + + def download(self, url, url_filename, filename, input_filter, auth=None, headers=None, version=None): + """ download ruleset file + :param url: download url + :param url_filename: if provided the filename within the (packet) resource + :param filename: target filename + :param input_filter: filter to use on received data before save + :param auth: authentication + :param headers: headers to send + :param version: version hash + """ + frm_url = url.replace('//', '/').replace(':/', '://') + file_stream = self.fetch(url=url, auth=auth, headers=headers) + if file_stream is not None: + try: + target_filename = '%s/%s' % (self._target_dir, filename) + if version: + save_data = "#@opnsense_download_hash:%s\n" % version + else: + save_data = "" + save_data += self._unpack(file_stream, url, url_filename) + save_data = self.filter(save_data, input_filter) + open(target_filename, 'w', buffering=10240).write(save_data) + except IOError: + syslog.syslog(syslog.LOG_ERR, 'cannot write to %s' % target_filename) + return None + syslog.syslog(syslog.LOG_INFO, 'download completed for %s' % frm_url) + else: + syslog.syslog(syslog.LOG_ERR, 'download failed for %s' % frm_url) @staticmethod - def is_supported(proto): + def is_supported(url): """ check if protocol is supported - :param proto: - :return: + :param url: uri to request resource from + :return: bool """ - if proto in ['http', 'https']: + if str(url).split(':')[0].lower() in ['http', 'https']: return True else: return False diff --git a/src/opnsense/scripts/suricata/lib/metadata.py b/src/opnsense/scripts/suricata/lib/metadata.py index 91fb5aa49..dc4e67d21 100644 --- a/src/opnsense/scripts/suricata/lib/metadata.py +++ b/src/opnsense/scripts/suricata/lib/metadata.py @@ -108,7 +108,10 @@ class Metadata(object): else: metadata_record['url'] = ('%s/%s' % (metadata_record['source']['url'], metadata_record['filename'])) - + if rule_xml.find('version') is not None and 'url' in rule_xml.find('version').attrib: + metadata_record['version_url'] = rule_xml.find('version').attrib['url'] + else: + metadata_record['version_url'] = None if 'prefix' in src_location.attrib: description_prefix = "%s/" % src_location.attrib['prefix'] else: diff --git a/src/opnsense/scripts/suricata/rule-updater.py b/src/opnsense/scripts/suricata/rule-updater.py index d4276c8cd..2b7ee353c 100755 --- a/src/opnsense/scripts/suricata/rule-updater.py +++ b/src/opnsense/scripts/suricata/rule-updater.py @@ -27,11 +27,12 @@ -------------------------------------------------------------------------------------- - update suricata rules + update/download suricata rules """ import os import sys +import syslog import fcntl from ConfigParser import ConfigParser from lib import metadata @@ -73,7 +74,7 @@ if __name__ == '__main__': for rule in md.list_rules(rule_properties): if 'url' in rule['source']: download_proto = str(rule['source']['url']).split(':')[0].lower() - if dl.is_supported(download_proto): + if dl.is_supported(url=rule['source']['url']): if rule['filename'] not in enabled_rulefiles: try: # remove configurable but unselected file @@ -86,6 +87,13 @@ if __name__ == '__main__': auth = (rule['source']['username'], rule['source']['password']) else: auth = None - dl.download(proto=download_proto, url=rule['url'], url_filename=rule['url_filename'], - filename=rule['filename'], input_filter=input_filter, auth=auth, - headers=rule['http_headers']) + # when metadata supports versioning, check if either version or settings changed before download + remote_hash = dl.fetch_version_hash(check_url=rule['version_url'], input_filter=input_filter, + auth=auth, headers=rule['http_headers']) + local_hash = dl.installed_file_hash(rule['filename']) + if remote_hash is None or remote_hash != local_hash: + dl.download(url=rule['url'], url_filename=rule['url_filename'], + filename=rule['filename'], input_filter=input_filter, auth=auth, + headers=rule['http_headers'], version=remote_hash) + else: + syslog.syslog(syslog.LOG_INFO, 'download skipped %s, same version' % rule['filename'])