diff --git a/src/opnsense/scripts/suricata/lib/metadata.py b/src/opnsense/scripts/suricata/lib/metadata.py index a75c19393..6176e2867 100755 --- a/src/opnsense/scripts/suricata/lib/metadata.py +++ b/src/opnsense/scripts/suricata/lib/metadata.py @@ -50,6 +50,7 @@ class Metadata(object): if xml_data.find(search_tag) > -1: xml_data = xml_data.replace(search_tag, replace_tags[tag]) rule_xml = xml.etree.ElementTree.fromstring(xml_data) + rule_xml.attrib['metadata_source'] = os.path.basename(filename) yield rule_xml except xml.etree.ElementTree.ParseError: # unparseable metadata @@ -87,6 +88,8 @@ class Metadata(object): if rule_xml.find('headers') is not None: for header in rule_xml.find('headers'): http_headers[header.tag] = header.text.strip() + + required_files = list() for rule_filename in rule_xml.find('files'): if 'documentation_url' in rule_filename.attrib: documentation_url = rule_filename.attrib['documentation_url'] @@ -94,7 +97,7 @@ class Metadata(object): documentation_url = rule_xml.attrib['documentation_url'] else: documentation_url = "" - metadata_record = dict() + metadata_record = {'required': False, 'metadata_source': rule_xml.attrib['metadata_source']} metadata_record['documentation_url'] = documentation_url metadata_record['source'] = src_location.attrib metadata_record['filename'] = rule_filename.text.strip() @@ -124,5 +127,14 @@ class Metadata(object): metadata_record['description'] = '%s%s' % (description_prefix, rule_filename.text) if metadata_record['filename'] not in target_filenames: - yield metadata_record + if 'required' in rule_filename.attrib \ + and rule_filename.attrib['required'].lower().strip() == 'true': + # collect required rules/files, flush when this metadata package is parsed + metadata_record['required'] = True + required_files.append(metadata_record) + else: + yield metadata_record target_filenames.append(metadata_record['filename']) + # flush required files last, so we can skip required when there's nothing else in the set selected + for metadata_record in required_files: + yield metadata_record diff --git a/src/opnsense/scripts/suricata/listInstallableRulesets.py b/src/opnsense/scripts/suricata/listInstallableRulesets.py index 226233bf9..965a76f85 100755 --- a/src/opnsense/scripts/suricata/listInstallableRulesets.py +++ b/src/opnsense/scripts/suricata/listInstallableRulesets.py @@ -42,12 +42,13 @@ if __name__ == '__main__': # (filenames should be unique) items = dict() for rule in md.list_rules(): - items[rule['filename']] = rule - rule_filename = ('%s/%s' % (rule_source_directory, rule['filename'])).replace('//', '/') - if os.path.exists(rule_filename): - items[rule['filename']]['modified_local'] = os.stat(rule_filename).st_mtime - else: - items[rule['filename']]['modified_local'] = None + if not rule['required']: + items[rule['filename']] = rule + rule_filename = ('%s/%s' % (rule_source_directory, rule['filename'])).replace('//', '/') + if os.path.exists(rule_filename): + items[rule['filename']]['modified_local'] = os.stat(rule_filename).st_mtime + else: + items[rule['filename']]['modified_local'] = None result = {'items': items, 'count': len(items)} result['properties'] = md.list_rule_properties() print (ujson.dumps(result)) diff --git a/src/opnsense/scripts/suricata/rule-updater.py b/src/opnsense/scripts/suricata/rule-updater.py index cca3ea6ec..2fdf66c24 100755 --- a/src/opnsense/scripts/suricata/rule-updater.py +++ b/src/opnsense/scripts/suricata/rule-updater.py @@ -51,6 +51,7 @@ if __name__ == '__main__': # load list of configured rules from generated config enabled_rulefiles = dict() rule_properties = dict() + metadata_sources = dict() updater_conf = '/usr/local/etc/suricata/rule-updater.config' if os.path.exists(updater_conf): cnf = ConfigParser() @@ -72,9 +73,15 @@ if __name__ == '__main__': md = metadata.Metadata() dl = downloader.Downloader(target_dir=rule_source_directory) for rule in md.list_rules(rule_properties): + if rule['metadata_source'] not in metadata_sources: + metadata_sources[rule['metadata_source']] = 0 if 'url' in rule['source']: download_proto = str(rule['source']['url']).split(':')[0].lower() if dl.is_supported(url=rule['source']['url']): + if rule['required'] and metadata_sources[rule['metadata_source']] > 0: + # Required files are always sorted last in list_rules(), add required when there's at least one + # file selected from the metadata package. + enabled_rulefiles[rule['filename']] = {'filter': ''} if rule['filename'] not in enabled_rulefiles: full_path = ('%s/%s' % (rule_source_directory, rule['filename'])).replace('//', '/') if os.path.isfile(full_path): @@ -93,6 +100,8 @@ if __name__ == '__main__': dl.download(url=rule['url'], url_filename=rule['url_filename'], filename=rule['filename'], input_filter=input_filter, auth=auth, headers=rule['http_headers'], version=remote_hash) + # count number of downloaded files/rules from this metadata package + metadata_sources[rule['metadata_source']] += 1 else: syslog.syslog(syslog.LOG_INFO, 'download skipped %s, same version' % rule['filename'])