diff --git a/src/opnsense/scripts/suricata/dropAlertLog.py b/src/opnsense/scripts/suricata/dropAlertLog.py index 2c6ad3fc5..2650d55e0 100755 --- a/src/opnsense/scripts/suricata/dropAlertLog.py +++ b/src/opnsense/scripts/suricata/dropAlertLog.py @@ -1,7 +1,7 @@ -#!/usr/local/bin/python2.7 +#!/usr/local/bin/python3.6 """ - Copyright (c) 2016 Ad Schellevis + Copyright (c) 2016-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -45,4 +45,4 @@ if __name__ == '__main__' and len(sys.argv) > 1: else: # archive, remove os.remove(filename) - print ("removed %s" % filename) + print("removed %s" % filename) diff --git a/src/opnsense/scripts/suricata/installRules.py b/src/opnsense/scripts/suricata/installRules.py index 82980f958..416076399 100755 --- a/src/opnsense/scripts/suricata/installRules.py +++ b/src/opnsense/scripts/suricata/installRules.py @@ -1,7 +1,7 @@ -#!/usr/local/bin/python2.7 +#!/usr/local/bin/python3.6 """ - Copyright (c) 2015 Ad Schellevis + Copyright (c) 2015-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -81,10 +81,10 @@ if __name__ == '__main__': # write data to file all_installed_files.append(filename.split('/')[-1]) - open('%s/%s' % (rule_target_dir, filename.split('/')[-1]), 'wb').write('\n'.join(output_data)) + open('%s/%s' % (rule_target_dir, filename.split('/')[-1]), 'w').write('\n'.join(output_data)) # flush all written rule filenames into yaml file - with open(rule_yaml_list, 'wb') as f_out: + with open(rule_yaml_list, 'w') as f_out: f_out.write('%YAML 1.1\n') f_out.write('---\n') f_out.write('rule-files:\n') diff --git a/src/opnsense/scripts/suricata/lib/__init__.py b/src/opnsense/scripts/suricata/lib/__init__.py index 0dda6b6e3..902edaa92 100755 --- a/src/opnsense/scripts/suricata/lib/__init__.py +++ b/src/opnsense/scripts/suricata/lib/__init__.py @@ -1,5 +1,5 @@ """ - Copyright (c) 2015 Ad Schellevis + Copyright (c) 2015-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/src/opnsense/scripts/suricata/lib/downloader.py b/src/opnsense/scripts/suricata/lib/downloader.py index 2ab4f62a1..d7116c6b4 100755 --- a/src/opnsense/scripts/suricata/lib/downloader.py +++ b/src/opnsense/scripts/suricata/lib/downloader.py @@ -80,7 +80,7 @@ class Downloader(object): :param src: handle to temp file :param source_filename: original source filename :param filename: filename to extract - :return: text + :return: string """ src.seek(0) unpack_type=None @@ -112,9 +112,9 @@ class Downloader(object): rule_content.append(zf.open(item).read()) elif filename is None and item.file_size > 0 and item.filename.lower().endswith('.rules'): rule_content.append(zf.open(item).read()) - return '\n'.join(rule_content) + return '\n'.join([x.decode() for x in rule_content]) else: - return src.read() + return src.read().decode() def fetch(self, url, auth=None, headers=None): """ Fetch file from remote location and save to temp, return filehandle pointed to start of temp file. @@ -178,12 +178,12 @@ class Downloader(object): if self.is_supported(check_url): version_fetch = self.fetch(url=check_url, auth=auth, headers=headers) if version_fetch: - version_response = version_fetch['handle'].read() + version_response = version_fetch['handle'].read().decode() hash_value = [json.dumps(input_filter), json.dumps(auth), json.dumps(headers), version_response] if not version_fetch['cached']: syslog.syslog(syslog.LOG_NOTICE, 'version response for %s : %s' % (check_url, version_response)) - return hashlib.md5('\n'.join(hash_value)).hexdigest() + return hashlib.md5(('\n'.join(hash_value)).encode()).hexdigest() return None def installed_file_hash(self, filename): diff --git a/src/opnsense/scripts/suricata/lib/metadata.py b/src/opnsense/scripts/suricata/lib/metadata.py index 6176e2867..8d0d0dbe0 100755 --- a/src/opnsense/scripts/suricata/lib/metadata.py +++ b/src/opnsense/scripts/suricata/lib/metadata.py @@ -1,5 +1,5 @@ """ - Copyright (c) 2015 Ad Schellevis + Copyright (c) 2015-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -44,8 +44,8 @@ class Metadata(object): """ for filename in sorted(glob.glob('%s*.xml' % self._rules_dir), reverse=True): try: - xml_data = open(filename).read() - for tag in replace_tags.keys(): + xml_data = open(filename, 'r').read() + for tag in replace_tags: search_tag = '%%%%%s%%%%' % tag if xml_data.find(search_tag) > -1: xml_data = xml_data.replace(search_tag, replace_tags[tag]) diff --git a/src/opnsense/scripts/suricata/lib/rulecache.py b/src/opnsense/scripts/suricata/lib/rulecache.py index 44fd80a58..cbf3b52e6 100755 --- a/src/opnsense/scripts/suricata/lib/rulecache.py +++ b/src/opnsense/scripts/suricata/lib/rulecache.py @@ -1,5 +1,5 @@ """ - Copyright (c) 2015 Ad Schellevis + Copyright (c) 2015-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -34,7 +34,7 @@ import glob import sqlite3 import shlex import fcntl -from ConfigParser import ConfigParser +from configparser import ConfigParser from lib import rule_source_directory @@ -79,51 +79,51 @@ class RuleCache(object): :param filename: :return: """ - data = open(filename) - for rule in data.read().split('\n'): - rule_info_record = {'rule': rule, 'metadata': None} - if rule.find('msg:') != -1: - # define basic record - record = {'enabled': True, 'source': filename.split('/')[-1]} - if rule.strip()[0] == '#': - record['enabled'] = False - record['action'] = rule.strip()[1:].split(' ')[0].replace('#', '') - else: - record['action'] = rule.strip().split(' ')[0] + with open(filename, 'r') as f_in: + for rule in f_in: + rule_info_record = {'rule': rule.strip(), 'metadata': None} + if rule.find('msg:') != -1: + # define basic record + record = {'enabled': True, 'source': filename.split('/')[-1]} + if rule.strip()[0] == '#': + record['enabled'] = False + record['action'] = rule.strip()[1:].split(' ')[0].replace('#', '') + else: + record['action'] = rule.strip().split(' ')[0] - rule_metadata = rule[rule.find('msg:'):-1] - for field in rule_metadata.split(';'): - fieldname = field[0:field.find(':')].strip() - fieldcontent = field[field.find(':') + 1:].strip() - if fieldname in self._rule_fields: - if fieldcontent[0] == '"': - content = fieldcontent[1:-1] - else: - content = fieldcontent + rule_metadata = rule[rule.find('msg:'):-1] + for field in rule_metadata.split(';'): + fieldname = field[0:field.find(':')].strip() + fieldcontent = field[field.find(':') + 1:].strip() + if fieldname in self._rule_fields: + if fieldcontent[0] == '"': + content = fieldcontent[1:-1] + else: + content = fieldcontent - if fieldname in record: - # if same field repeats, put items in list - if type(record[fieldname]) != list: - record[fieldname] = [record[fieldname]] - record[fieldname].append(content) - else: - record[fieldname] = content + if fieldname in record: + # if same field repeats, put items in list + if type(record[fieldname]) != list: + record[fieldname] = [record[fieldname]] + record[fieldname].append(content) + else: + record[fieldname] = content - for rule_field in self._rule_fields: - if rule_field not in record: - if rule_field in self._rule_defaults: - record[rule_field] = self._rule_defaults[rule_field] - else: - record[rule_field] = None + for rule_field in self._rule_fields: + if rule_field not in record: + if rule_field in self._rule_defaults: + record[rule_field] = self._rule_defaults[rule_field] + else: + record[rule_field] = None - # perform type conversions - for fieldname in record: - if type(record[fieldname]) == list: - record[fieldname] = '\n'.join(record[fieldname]) + # perform type conversions + for fieldname in record: + if type(record[fieldname]) == list: + record[fieldname] = '\n'.join(record[fieldname]) - rule_info_record['metadata'] = record + rule_info_record['metadata'] = record - yield rule_info_record + yield rule_info_record def is_changed(self): """ check if rules on disk are probably different from rules in cache @@ -170,7 +170,7 @@ class RuleCache(object): os.remove(self.cachefile) db = sqlite3.connect(self.cachefile) - db.text_factory = lambda x: unicode(x, 'utf-8', 'ignore') + db.text_factory = lambda x: str(x, 'utf-8', 'ignore') cur = db.cursor() cur.execute("create table stats (timestamp number, files number)") @@ -257,7 +257,7 @@ class RuleCache(object): sql += ' and ( ' else: sql += ' where ( ' - for fieldname in map(lambda x: x.lower().strip(), fieldnames.split(',')): + for fieldname in [x.lower().strip() for x in fieldnames.split(',')]: if fieldname in self._rule_fields or fieldname in additional_search_fields: if fieldname != fieldnames.split(',')[0].strip(): sql += ' or ' diff --git a/src/opnsense/scripts/suricata/listAlertLogs.py b/src/opnsense/scripts/suricata/listAlertLogs.py index 3716b820f..980f225ad 100755 --- a/src/opnsense/scripts/suricata/listAlertLogs.py +++ b/src/opnsense/scripts/suricata/listAlertLogs.py @@ -1,7 +1,7 @@ -#!/usr/local/bin/python2.7 +#!/usr/local/bin/python3.6 """ - Copyright (c) 2015 Ad Schellevis + Copyright (c) 2015-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/src/opnsense/scripts/suricata/listClasstypes.py b/src/opnsense/scripts/suricata/listClasstypes.py index 04930fb1e..3ef8e404f 100755 --- a/src/opnsense/scripts/suricata/listClasstypes.py +++ b/src/opnsense/scripts/suricata/listClasstypes.py @@ -1,7 +1,7 @@ -#!/usr/local/bin/python2.7 +#!/usr/local/bin/python3.6 """ - Copyright (c) 2015 Ad Schellevis + Copyright (c) 2015-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -41,4 +41,4 @@ if __name__ == '__main__': items = rc.list_class_types() result = {'items': items, 'count': len(items)} - print (ujson.dumps(result)) + print(ujson.dumps(result)) diff --git a/src/opnsense/scripts/suricata/listInstallableRulesets.py b/src/opnsense/scripts/suricata/listInstallableRulesets.py index 965a76f85..7b602fba5 100755 --- a/src/opnsense/scripts/suricata/listInstallableRulesets.py +++ b/src/opnsense/scripts/suricata/listInstallableRulesets.py @@ -1,7 +1,7 @@ -#!/usr/local/bin/python2.7 +#!/usr/local/bin/python3.6 """ - Copyright (c) 2015 Ad Schellevis + Copyright (c) 2015-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -51,4 +51,4 @@ if __name__ == '__main__': items[rule['filename']]['modified_local'] = None result = {'items': items, 'count': len(items)} result['properties'] = md.list_rule_properties() - print (ujson.dumps(result)) + print(ujson.dumps(result)) diff --git a/src/opnsense/scripts/suricata/queryAlertLog.py b/src/opnsense/scripts/suricata/queryAlertLog.py index 6d57aff31..c320f2c59 100755 --- a/src/opnsense/scripts/suricata/queryAlertLog.py +++ b/src/opnsense/scripts/suricata/queryAlertLog.py @@ -1,7 +1,7 @@ -#!/usr/local/bin/python2.7 +#!/usr/local/bin/python3.6 """ - Copyright (c) 2015 Ad Schellevis + Copyright (c) 2015-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/src/opnsense/scripts/suricata/queryInstalledRules.py b/src/opnsense/scripts/suricata/queryInstalledRules.py index a7bdc8163..a049f3d64 100755 --- a/src/opnsense/scripts/suricata/queryInstalledRules.py +++ b/src/opnsense/scripts/suricata/queryInstalledRules.py @@ -1,7 +1,7 @@ -#!/usr/local/bin/python2.7 +#!/usr/local/bin/python3.6 """ - Copyright (c) 2015 Ad Schellevis + Copyright (c) 2015-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -59,4 +59,4 @@ if __name__ == '__main__': # dump output result = rc.search(**parameters) result['parameters'] = parameters - print (ujson.dumps(result)) + print(ujson.dumps(result)) diff --git a/src/opnsense/scripts/suricata/rule-updater.py b/src/opnsense/scripts/suricata/rule-updater.py index 57b2034b8..3fc0a8cf9 100755 --- a/src/opnsense/scripts/suricata/rule-updater.py +++ b/src/opnsense/scripts/suricata/rule-updater.py @@ -1,7 +1,7 @@ -#!/usr/local/bin/python2.7 +#!/usr/local/bin/python3.6 """ - Copyright (c) 2015-2018 Ad Schellevis + Copyright (c) 2015-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -34,7 +34,7 @@ import os import sys import syslog import fcntl -from ConfigParser import ConfigParser +from configparser import ConfigParser from lib import metadata from lib import downloader from lib import rule_source_directory @@ -105,7 +105,7 @@ if __name__ == '__main__': syslog.syslog(syslog.LOG_INFO, 'download skipped %s, same version' % rule['filename']) # cleanup: match all installed rulesets against the configured ones and remove uninstalled rules - md_filenames = map(lambda x:x['filename'], md.list_rules(rule_properties)) + md_filenames = [x['filename'] for x in md.list_rules(rule_properties)] for filename in enabled_rulefiles: full_path = ('%s/%s' % (rule_source_directory, filename)).replace('//', '/') if filename not in md_filenames and os.path.isfile(full_path):