From 409dd5f10a148c8cc92903ad9bf2d3daccda0c55 Mon Sep 17 00:00:00 2001 From: Ad Schellevis Date: Tue, 30 Jun 2015 20:21:20 +0200 Subject: [PATCH] (ids) work in progress rule file download --- .../scripts/suricata/lib/downloader.py | 63 ++++++++++++++++ src/opnsense/scripts/suricata/lib/metadata.py | 71 +++++++++++++++++++ .../scripts/suricata/listInstallableRules.py | 44 ++++++++++++ .../suricata/metadata/rules/et-open.xml | 9 +++ src/opnsense/scripts/suricata/rule-updater.py | 66 +++++++++++++++++ 5 files changed, 253 insertions(+) create mode 100644 src/opnsense/scripts/suricata/lib/downloader.py create mode 100644 src/opnsense/scripts/suricata/lib/metadata.py create mode 100755 src/opnsense/scripts/suricata/listInstallableRules.py create mode 100644 src/opnsense/scripts/suricata/metadata/rules/et-open.xml create mode 100644 src/opnsense/scripts/suricata/rule-updater.py diff --git a/src/opnsense/scripts/suricata/lib/downloader.py b/src/opnsense/scripts/suricata/lib/downloader.py new file mode 100644 index 000000000..5dc0df925 --- /dev/null +++ b/src/opnsense/scripts/suricata/lib/downloader.py @@ -0,0 +1,63 @@ +""" + Copyright (c) 2015 Ad Schellevis + + part of OPNsense (https://www.opnsense.org/) + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------------- + rule downloader module (may need to be extended in the future, this version only processes http(s)) +""" +import syslog +import requests + +class Downloader(object): + def __init__(self, target_dir): + self._target_dir = target_dir + + def download(self, proto, url): + if proto in ('http','https'): + frm_url = url.replace('//','/').replace(':/','://') + req = requests.get(url=frm_url) + if req.status_code == 200: + target_filename = ('%s/%s'%(self._target_dir, frm_url.split('/')[-1])).replace('//','/') + try: + open(target_filename,'wb').write(req.text) + except IOError: + syslog.syslog(syslog.LOG_ERR, 'cannot write to %s'%(target_filename)) + return None + syslog.syslog(syslog.LOG_INFO, 'download completed for %s'%(frm_url)) + else: + syslog.syslog(syslog.LOG_ERR, 'download failed for %s'%(frm_url)) + + @staticmethod + def is_supported(proto): + """ check if protocol is supported + :param proto: + :return: + """ + if proto in ['http','https']: + return True + else: + return False diff --git a/src/opnsense/scripts/suricata/lib/metadata.py b/src/opnsense/scripts/suricata/lib/metadata.py new file mode 100644 index 000000000..0302fcbc9 --- /dev/null +++ b/src/opnsense/scripts/suricata/lib/metadata.py @@ -0,0 +1,71 @@ +""" + Copyright (c) 2015 Ad Schellevis + + part of OPNsense (https://www.opnsense.org/) + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------------- + shared module for suricata rule metadata +""" +import os +import syslog +import glob +import xml.etree.ElementTree + +class Metadata(object): + def __init__(self): + self._rules_dir = '%s/../metadata/rules/'%(os.path.dirname(os.path.abspath(__file__))) + + def list_rules(self): + """ list all available rules + :return: generator method returning all known rulefiles + """ + for filename in sorted(glob.glob('%s*.xml'%self._rules_dir)): + try: + ruleXML=xml.etree.ElementTree.fromstring(open(filename).read()) + except xml.etree.ElementTree.ParseError: + # unparseable metadata + syslog.syslog(syslog.LOG_ERR,'suricata metadata unparsable @ %s'%filename) + continue + + src_location = ruleXML.find('location') + if src_location is None or 'url' not in src_location.attrib: + syslog.syslog(syslog.LOG_ERR,'suricata metadata missing location @ %s'%filename) + else: + if ruleXML.find('files') is None: + syslog.syslog(syslog.LOG_ERR,'suricata metadata missing files @ %s'%filename) + else: + for rule_filename in ruleXML.find('files'): + metadata_record = dict() + metadata_record['source'] = src_location.attrib + metadata_record['filename'] = rule_filename.text.strip() + if 'description' in rule_filename.attrib: + metadata_record['description'] = rule_filename.attrib['description'] + else: + metadata_record['description'] = rule_filename.text + + yield metadata_record + + diff --git a/src/opnsense/scripts/suricata/listInstallableRules.py b/src/opnsense/scripts/suricata/listInstallableRules.py new file mode 100755 index 000000000..d602bdd3a --- /dev/null +++ b/src/opnsense/scripts/suricata/listInstallableRules.py @@ -0,0 +1,44 @@ +#!/usr/local/bin/python2.7 +""" + Copyright (c) 2015 Ad Schellevis + + part of OPNsense (https://www.opnsense.org/) + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------------- + list downloadable/installable suricata rules, see metadata/rules/*.xml +""" +from lib import metadata +import ujson + +md = metadata.Metadata() +if __name__ == '__main__': + # collect all installable rules indexed by (target) filename + # (filenames should be unique) + items = dict() + for rule in md.list_rules(): + items[rule['filename']] = rule + result = {'items': items, 'count':len(items)} + print (ujson.dumps(result)) diff --git a/src/opnsense/scripts/suricata/metadata/rules/et-open.xml b/src/opnsense/scripts/suricata/metadata/rules/et-open.xml new file mode 100644 index 000000000..1bcaf1203 --- /dev/null +++ b/src/opnsense/scripts/suricata/metadata/rules/et-open.xml @@ -0,0 +1,9 @@ + + + + + emerging-shellcode.rules + emerging-smtp.rules + emerging-sql.rules + + \ No newline at end of file diff --git a/src/opnsense/scripts/suricata/rule-updater.py b/src/opnsense/scripts/suricata/rule-updater.py new file mode 100644 index 000000000..b9198e0fb --- /dev/null +++ b/src/opnsense/scripts/suricata/rule-updater.py @@ -0,0 +1,66 @@ +#!/usr/local/bin/python2.7 +""" + Copyright (c) 2015 Ad Schellevis + + part of OPNsense (https://www.opnsense.org/) + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------------- + update suricata rules +""" +import os +from ConfigParser import ConfigParser +from lib import metadata +from lib import downloader + +if __name__ == '__main__': + # load list of configured rules from generated config + enabled_rulefiles=[] + updater_conf='/usr/local/etc/suricata/rule-updater.config' + target_directory='/usr/local/etc/suricata/rules/' + if os.path.exists(updater_conf): + cnf = ConfigParser() + cnf.read(updater_conf) + for section in cnf.sections(): + if cnf.has_option(section,'enabled') and cnf.getint(section,'enabled') == 1: + enabled_rulefiles.append(section.strip()) + + # download / remove rules + md = metadata.Metadata() + dl = downloader.Downloader(target_dir=target_directory) + for rule in md.list_rules(): + if 'url' in rule['source']: + download_proto=str(rule['source']['url']).split(':')[0].lower() + if dl.is_supported(download_proto): + if rule['filename'] not in enabled_rulefiles: + try: + # remove configurable but unselected file + os.remove(('%s/%s'%(target_directory, rule['filename'])).replace('//', '/')) + except: + pass + else: + url = ('%s/%s'%(rule['source']['url'],rule['filename'])) + dl.download(proto=download_proto, url=url) +