lucaspalomodevelop/core
1
0
Fork 0
mirror of https://github.com/lucaspalomodevelop/core.git synced 2026-03-13 16:14:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

IDS, support required rules/files in metadata package, closes https://github.com/opnsense/core/issues/3228

Browse Source
This commit is contained in:
Ad Schellevis 2019-02-12 21:22:09 +01:00
parent 8695737563
commit 7b758564cf
3 changed files with 30 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/opnsense/scripts/suricata/lib/metadata.py
View File

@ -50,6 +50,7 @@ class Metadata(object):
if xml_data.find(search_tag) > -1:
xml_data = xml_data.replace(search_tag, replace_tags[tag])
rule_xml = xml.etree.ElementTree.fromstring(xml_data)
rule_xml.attrib['metadata_source'] = os.path.basename(filename)
yield rule_xml
except xml.etree.ElementTree.ParseError:
# unparseable metadata
@ -87,6 +88,8 @@ class Metadata(object):
if rule_xml.find('headers') is not None:
for header in rule_xml.find('headers'):
http_headers[header.tag] = header.text.strip()
required_files = list()
for rule_filename in rule_xml.find('files'):
if 'documentation_url' in rule_filename.attrib:
documentation_url = rule_filename.attrib['documentation_url']
@ -94,7 +97,7 @@ class Metadata(object):
documentation_url = rule_xml.attrib['documentation_url']
else:
documentation_url = ""
metadata_record = dict()
metadata_record = {'required': False, 'metadata_source': rule_xml.attrib['metadata_source']}
metadata_record['documentation_url'] = documentation_url
metadata_record['source'] = src_location.attrib
metadata_record['filename'] = rule_filename.text.strip()
@ -124,5 +127,14 @@ class Metadata(object):
metadata_record['description'] = '%s%s' % (description_prefix,
rule_filename.text)
if metadata_record['filename'] not in target_filenames:
yield metadata_record
if 'required' in rule_filename.attrib \
and rule_filename.attrib['required'].lower().strip() == 'true':
# collect required rules/files, flush when this metadata package is parsed
metadata_record['required'] = True
required_files.append(metadata_record)
else:
yield metadata_record
target_filenames.append(metadata_record['filename'])
# flush required files last, so we can skip required when there's nothing else in the set selected
for metadata_record in required_files:
yield metadata_record

13
src/opnsense/scripts/suricata/listInstallableRulesets.py
View File

@ -42,12 +42,13 @@ if __name__ == '__main__':
# (filenames should be unique)
items = dict()
for rule in md.list_rules():
items[rule['filename']] = rule
rule_filename = ('%s/%s' % (rule_source_directory, rule['filename'])).replace('//', '/')
if os.path.exists(rule_filename):
items[rule['filename']]['modified_local'] = os.stat(rule_filename).st_mtime
else:
items[rule['filename']]['modified_local'] = None
if not rule['required']:
items[rule['filename']] = rule
rule_filename = ('%s/%s' % (rule_source_directory, rule['filename'])).replace('//', '/')
if os.path.exists(rule_filename):
items[rule['filename']]['modified_local'] = os.stat(rule_filename).st_mtime
else:
items[rule['filename']]['modified_local'] = None
result = {'items': items, 'count': len(items)}
result['properties'] = md.list_rule_properties()
print (ujson.dumps(result))

9
src/opnsense/scripts/suricata/rule-updater.py
View File

@ -51,6 +51,7 @@ if __name__ == '__main__':
# load list of configured rules from generated config
enabled_rulefiles = dict()
rule_properties = dict()
metadata_sources = dict()
updater_conf = '/usr/local/etc/suricata/rule-updater.config'
if os.path.exists(updater_conf):
cnf = ConfigParser()
@ -72,9 +73,15 @@ if __name__ == '__main__':
md = metadata.Metadata()
dl = downloader.Downloader(target_dir=rule_source_directory)
for rule in md.list_rules(rule_properties):
if rule['metadata_source'] not in metadata_sources:
metadata_sources[rule['metadata_source']] = 0
if 'url' in rule['source']:
download_proto = str(rule['source']['url']).split(':')[0].lower()
if dl.is_supported(url=rule['source']['url']):
if rule['required'] and metadata_sources[rule['metadata_source']] > 0:
# Required files are always sorted last in list_rules(), add required when there's at least one
# file selected from the metadata package.
enabled_rulefiles[rule['filename']] = {'filter': ''}
if rule['filename'] not in enabled_rulefiles:
full_path = ('%s/%s' % (rule_source_directory, rule['filename'])).replace('//', '/')
if os.path.isfile(full_path):
@ -93,6 +100,8 @@ if __name__ == '__main__':
dl.download(url=rule['url'], url_filename=rule['url_filename'],
filename=rule['filename'], input_filter=input_filter, auth=auth,
headers=rule['http_headers'], version=remote_hash)
# count number of downloaded files/rules from this metadata package
metadata_sources[rule['metadata_source']] += 1
else:
syslog.syslog(syslog.LOG_INFO, 'download skipped %s, same version' % rule['filename'])