mirror of
https://github.com/lucaspalomodevelop/core.git
synced 2026-03-16 09:34:39 +00:00
(ids) work in progress rule file download
This commit is contained in:
Ad Schellevis
parent
67e2e3e2a5
commit
409dd5f10a
63
src/opnsense/scripts/suricata/lib/downloader.py
Normal file
63
src/opnsense/scripts/suricata/lib/downloader.py
Normal file
@ -0,0 +1,63 @@
|
||||
"""
|
||||
Copyright (c) 2015 Ad Schellevis
|
||||
|
||||
part of OPNsense (https://www.opnsense.org/)
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
||||
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
||||
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
|
||||
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------------
|
||||
rule downloader module (may need to be extended in the future, this version only processes http(s))
|
||||
"""
|
||||
import syslog
|
||||
import requests
|
||||
|
||||
class Downloader(object):
|
||||
def __init__(self, target_dir):
|
||||
self._target_dir = target_dir
|
||||
|
||||
def download(self, proto, url):
|
||||
if proto in ('http','https'):
|
||||
frm_url = url.replace('//','/').replace(':/','://')
|
||||
req = requests.get(url=frm_url)
|
||||
if req.status_code == 200:
|
||||
target_filename = ('%s/%s'%(self._target_dir, frm_url.split('/')[-1])).replace('//','/')
|
||||
try:
|
||||
open(target_filename,'wb').write(req.text)
|
||||
except IOError:
|
||||
syslog.syslog(syslog.LOG_ERR, 'cannot write to %s'%(target_filename))
|
||||
return None
|
||||
syslog.syslog(syslog.LOG_INFO, 'download completed for %s'%(frm_url))
|
||||
else:
|
||||
syslog.syslog(syslog.LOG_ERR, 'download failed for %s'%(frm_url))
|
||||
|
||||
@staticmethod
|
||||
def is_supported(proto):
|
||||
""" check if protocol is supported
|
||||
:param proto:
|
||||
:return:
|
||||
"""
|
||||
if proto in ['http','https']:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
71
src/opnsense/scripts/suricata/lib/metadata.py
Normal file
71
src/opnsense/scripts/suricata/lib/metadata.py
Normal file
@ -0,0 +1,71 @@
|
||||
"""
|
||||
Copyright (c) 2015 Ad Schellevis
|
||||
|
||||
part of OPNsense (https://www.opnsense.org/)
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
||||
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
||||
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
|
||||
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------------
|
||||
shared module for suricata rule metadata
|
||||
"""
|
||||
import os
|
||||
import syslog
|
||||
import glob
|
||||
import xml.etree.ElementTree
|
||||
|
||||
class Metadata(object):
|
||||
def __init__(self):
|
||||
self._rules_dir = '%s/../metadata/rules/'%(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
def list_rules(self):
|
||||
""" list all available rules
|
||||
:return: generator method returning all known rulefiles
|
||||
"""
|
||||
for filename in sorted(glob.glob('%s*.xml'%self._rules_dir)):
|
||||
try:
|
||||
ruleXML=xml.etree.ElementTree.fromstring(open(filename).read())
|
||||
except xml.etree.ElementTree.ParseError:
|
||||
# unparseable metadata
|
||||
syslog.syslog(syslog.LOG_ERR,'suricata metadata unparsable @ %s'%filename)
|
||||
continue
|
||||
|
||||
src_location = ruleXML.find('location')
|
||||
if src_location is None or 'url' not in src_location.attrib:
|
||||
syslog.syslog(syslog.LOG_ERR,'suricata metadata missing location @ %s'%filename)
|
||||
else:
|
||||
if ruleXML.find('files') is None:
|
||||
syslog.syslog(syslog.LOG_ERR,'suricata metadata missing files @ %s'%filename)
|
||||
else:
|
||||
for rule_filename in ruleXML.find('files'):
|
||||
metadata_record = dict()
|
||||
metadata_record['source'] = src_location.attrib
|
||||
metadata_record['filename'] = rule_filename.text.strip()
|
||||
if 'description' in rule_filename.attrib:
|
||||
metadata_record['description'] = rule_filename.attrib['description']
|
||||
else:
|
||||
metadata_record['description'] = rule_filename.text
|
||||
|
||||
yield metadata_record
|
||||
|
||||
|
||||
44
src/opnsense/scripts/suricata/listInstallableRules.py
Executable file
44
src/opnsense/scripts/suricata/listInstallableRules.py
Executable file
@ -0,0 +1,44 @@
|
||||
#!/usr/local/bin/python2.7
|
||||
"""
|
||||
Copyright (c) 2015 Ad Schellevis
|
||||
|
||||
part of OPNsense (https://www.opnsense.org/)
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
||||
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
||||
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
|
||||
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------------
|
||||
list downloadable/installable suricata rules, see metadata/rules/*.xml
|
||||
"""
|
||||
from lib import metadata
|
||||
import ujson
|
||||
|
||||
md = metadata.Metadata()
|
||||
if __name__ == '__main__':
|
||||
# collect all installable rules indexed by (target) filename
|
||||
# (filenames should be unique)
|
||||
items = dict()
|
||||
for rule in md.list_rules():
|
||||
items[rule['filename']] = rule
|
||||
result = {'items': items, 'count':len(items)}
|
||||
print (ujson.dumps(result))
|
||||
9
src/opnsense/scripts/suricata/metadata/rules/et-open.xml
Normal file
9
src/opnsense/scripts/suricata/metadata/rules/et-open.xml
Normal file
@ -0,0 +1,9 @@
|
||||
<?xml version="1.0"?>
|
||||
<ruleset>
|
||||
<location url="https://rules.emergingthreats.net/open/suricata/rules/"/>
|
||||
<files>
|
||||
<file decription="emerging-shellcode">emerging-shellcode.rules</file>
|
||||
<file decription="emerging-smtp">emerging-smtp.rules</file>
|
||||
<file>emerging-sql.rules</file>
|
||||
</files>
|
||||
</ruleset>
|
||||
66
src/opnsense/scripts/suricata/rule-updater.py
Normal file
66
src/opnsense/scripts/suricata/rule-updater.py
Normal file
@ -0,0 +1,66 @@
|
||||
#!/usr/local/bin/python2.7
|
||||
"""
|
||||
Copyright (c) 2015 Ad Schellevis
|
||||
|
||||
part of OPNsense (https://www.opnsense.org/)
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
||||
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
||||
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
|
||||
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------------
|
||||
update suricata rules
|
||||
"""
|
||||
import os
|
||||
from ConfigParser import ConfigParser
|
||||
from lib import metadata
|
||||
from lib import downloader
|
||||
|
||||
if __name__ == '__main__':
|
||||
# load list of configured rules from generated config
|
||||
enabled_rulefiles=[]
|
||||
updater_conf='/usr/local/etc/suricata/rule-updater.config'
|
||||
target_directory='/usr/local/etc/suricata/rules/'
|
||||
if os.path.exists(updater_conf):
|
||||
cnf = ConfigParser()
|
||||
cnf.read(updater_conf)
|
||||
for section in cnf.sections():
|
||||
if cnf.has_option(section,'enabled') and cnf.getint(section,'enabled') == 1:
|
||||
enabled_rulefiles.append(section.strip())
|
||||
|
||||
# download / remove rules
|
||||
md = metadata.Metadata()
|
||||
dl = downloader.Downloader(target_dir=target_directory)
|
||||
for rule in md.list_rules():
|
||||
if 'url' in rule['source']:
|
||||
download_proto=str(rule['source']['url']).split(':')[0].lower()
|
||||
if dl.is_supported(download_proto):
|
||||
if rule['filename'] not in enabled_rulefiles:
|
||||
try:
|
||||
# remove configurable but unselected file
|
||||
os.remove(('%s/%s'%(target_directory, rule['filename'])).replace('//', '/'))
|
||||
except:
|
||||
pass
|
||||
else:
|
||||
url = ('%s/%s'%(rule['source']['url'],rule['filename']))
|
||||
dl.download(proto=download_proto, url=url)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user