From 565fd72bba228db672b4fdab2ef4374344876b1c Mon Sep 17 00:00:00 2001 From: Ad Schellevis Date: Tue, 27 Dec 2016 12:08:54 +0100 Subject: [PATCH] (ids) add support for inline configuration settings (subscription based url's for example), add basic auth support. Example supported format: snort.blacklist.rules --- Registers the setting "snort.oinkcode" which is used to construct the download url. This commit doesn't include definitions for new content, in case someone wants to create a definition file, it should be easy now :) --- .../OPNsense/IDS/Api/SettingsController.php | 70 ++++++++++ .../mvc/app/models/OPNsense/IDS/IDS.xml | 12 ++ .../mvc/app/views/OPNsense/IDS/index.volt | 128 +++++++++++------- .../scripts/suricata/lib/downloader.py | 10 +- src/opnsense/scripts/suricata/rule-updater.py | 6 +- .../OPNsense/IDS/rule-updater.config | 8 ++ 6 files changed, 180 insertions(+), 54 deletions(-) diff --git a/src/opnsense/mvc/app/controllers/OPNsense/IDS/Api/SettingsController.php b/src/opnsense/mvc/app/controllers/OPNsense/IDS/Api/SettingsController.php index 7b62c0372..1a4175a0d 100644 --- a/src/opnsense/mvc/app/controllers/OPNsense/IDS/Api/SettingsController.php +++ b/src/opnsense/mvc/app/controllers/OPNsense/IDS/Api/SettingsController.php @@ -266,6 +266,76 @@ class SettingsController extends ApiMutableModelControllerBase return $result; } + /** + * list ruleset properties + * @return array + */ + public function getRulesetpropertiesAction() + { + $result = array('properties' => array()); + $backend = new Backend(); + $response = $backend->configdRun("ids list installablerulesets"); + $data = json_decode($response, true); + if ($data != null && isset($data["properties"])) { + foreach ($data['properties'] as $key => $settings) { + $result['properties'][$key] = !empty($settings['default']) ? $settings['default'] : ""; + foreach ($this->getModel()->fileTags->tag->__items as $tag) { + if ((string)$tag->property == $key) { + $result['properties'][(string)$tag->property] = (string)$tag->value; + } + } + } + } + return $result; + } + + /** + * update ruleset properties + * @return array + */ + public function setRulesetpropertiesAction() + { + $result = array("result" => "failed"); + if ($this->request->isPost() && $this->request->hasPost("properties")) { + // only update properties available in "ids list installablerulesets" + $backend = new Backend(); + $response = $backend->configdRun("ids list installablerulesets"); + $data = json_decode($response, true); + if ($data != null && isset($data["properties"])) { + $setProperties = $this->request->getPost("properties"); + foreach ($setProperties as $key => $value) { + if (isset($data['properties'][$key])) { + if (!isset($result['fields'])) { + $result['fields'] = array(); // return updated fields + } + $result['fields'][] = $key; + $resultTag = null; + foreach ($this->getModel()->fileTags->tag->__items as $tag) { + if ((string)$tag->property == $key) { + $resultTag = $tag; + break; + } + } + if ($resultTag == null) { + $resultTag = $this->getModel()->fileTags->tag->Add(); + } + $resultTag->property = (string)$key; + $resultTag->value = (string)$value; + } + } + $validations = $this->getModel()->validate(); + if (count($validations)) { + $result['validations'] = $validations; + } else { + $this->getModel()->serializeToConfig(); + Config::getInstance()->save(); + $result["result"] = "saved"; + } + } + } + return $result; + } + /** * list all installable rules including current status * @return array|mixed list of items when $id is null otherwise the selected item is returned diff --git a/src/opnsense/mvc/app/models/OPNsense/IDS/IDS.xml b/src/opnsense/mvc/app/models/OPNsense/IDS/IDS.xml index 451987cf5..ffab75760 100644 --- a/src/opnsense/mvc/app/models/OPNsense/IDS/IDS.xml +++ b/src/opnsense/mvc/app/models/OPNsense/IDS/IDS.xml @@ -84,6 +84,18 @@ + + + + Y + /^([\t\n\v\f\r\- 0-9a-zA-Z.,_\x{00A0}-\x{FFFF}]){1,255}$/u + + + N + /^([\t\n\v\f\r\- 0-9a-zA-Z.,_\x{00A0}-\x{FFFF}]){1,255}$/u + + + 0 diff --git a/src/opnsense/mvc/app/views/OPNsense/IDS/index.volt b/src/opnsense/mvc/app/views/OPNsense/IDS/index.volt index e6da25a75..f18c1d11c 100644 --- a/src/opnsense/mvc/app/views/OPNsense/IDS/index.volt +++ b/src/opnsense/mvc/app/views/OPNsense/IDS/index.volt @@ -153,6 +153,7 @@ POSSIBILITY OF SUCH DAMAGE. if (status == "success" || data['status'].toLowerCase().trim() == "ok") { result_status = true; } + $('#scheduled_updates').show(); callback_funct(result_status); }); }); @@ -207,6 +208,65 @@ POSSIBILITY OF SUCH DAMAGE. $('a[data-toggle="tab"]').on('shown.bs.tab', function (e) { if (e.target.id == 'settings_tab'){ loadGeneralSettings(); + } else if (e.target.id == 'download_settings_tab') { + /** + * grid for installable rule files + */ + $('#grid-rule-files').bootgrid('destroy'); // always destroy previous grid, so data is always fresh + $("#grid-rule-files").UIBootgrid({ + search:'/api/ids/settings/listRulesets', + get:'/api/ids/settings/getRuleset/', + set:'/api/ids/settings/setRuleset/', + toggle:'/api/ids/settings/toggleRuleset/', + options:{ + navigation:0, + formatters:{ + rowtoggle: function (column, row) { + var toggle = " "; + if (parseInt(row[column.id], 2) == 1) { + toggle += ""; + } else { + toggle += ""; + } + return toggle; + } + }, + converters: { + // show "not installed" for rules without timestamp (not on disc) + rulets: { + from: function (value) { + return value; + }, + to: function (value) { + if ( value == null ) { + return "{{ lang._('not installed') }}"; + } else { + return value; + } + } + } + } + } + }); + // display file settings (if available) + ajaxGet(url="/api/ids/settings/getRulesetproperties", sendData={}, callback=function(data, status) { + if (status == "success") { + var rows = []; + // generate rows with field references + $.each(data['properties'], function(key, value) { + rows.push(''+key+''); + }); + $("#grid-rule-files-settings > tbody").html(rows.join('')); + // update with data + $(".rulesetprop").each(function(){ + $(this).val(data['properties'][$(this).data('id')]); + }); + if (rows.length > 0) { + $("#grid-rule-files-settings").parent().parent().show(); + $("#updateSettings").show(); + } + } + }); } else if (e.target.id == 'rule_tab'){ // // activate rule tab page @@ -275,46 +335,6 @@ POSSIBILITY OF SUCH DAMAGE. toggle:'/api/ids/settings/toggleUserRule/' } ); - } else if (e.target.id == 'download_settings_tab') { - /** - * grid for installable rule files - */ - $('#grid-rule-files').bootgrid('destroy'); // always destroy previous grid, so data is always fresh - $("#grid-rule-files").UIBootgrid({ - search:'/api/ids/settings/listRulesets', - get:'/api/ids/settings/getRuleset/', - set:'/api/ids/settings/setRuleset/', - toggle:'/api/ids/settings/toggleRuleset/', - options:{ - navigation:0, - formatters:{ - rowtoggle: function (column, row) { - var toggle = " "; - if (parseInt(row[column.id], 2) == 1) { - toggle += ""; - } else { - toggle += ""; - } - return toggle; - } - }, - converters: { - // show "not installed" for rules without timestamp (not on disc) - rulets: { - from: function (value) { - return value; - }, - to: function (value) { - if ( value == null ) { - return "{{ lang._('not installed') }}"; - } else { - return value; - } - } - } - } - } - }); } }) @@ -344,6 +364,16 @@ POSSIBILITY OF SUCH DAMAGE. } }); }); + $("#updateSettings").click(function(){ + $("#updateSettings_progress").addClass("fa fa-spinner fa-pulse"); + var settings = {}; + $(".rulesetprop").each(function(){ + settings[$(this).data('id')] = $(this).val(); + }); + ajaxCall(url="/api/ids/settings/setRulesetproperties", sendData={'properties': settings}, callback=function(data,status) { + $("#updateSettings_progress").removeClass("fa fa-spinner fa-pulse"); + }); + }); /** * update (userdefined) rules @@ -365,15 +395,7 @@ POSSIBILITY OF SUCH DAMAGE. // when done, disable progress animation and reload grid. $('#grid-rule-files').bootgrid('reload'); updateStatus(); - if ($('#scheduled_updates').is(':hidden') ){ - // save and reconfigure on initial download (tries to create a cron job) - actionReconfigure(function(status){ - loadGeneralSettings(); - $("#updateRulesAct_progress").removeClass("fa fa-spinner fa-pulse"); - }); - } else { - $("#updateRulesAct_progress").removeClass("fa fa-spinner fa-pulse"); - } + $("#updateRulesAct_progress").removeClass("fa fa-spinner fa-pulse"); }); }); @@ -586,20 +608,24 @@ POSSIBILITY OF SUCH DAMAGE. - +
{{ lang._('Settings') }}
- + + + +
+
{{ lang._('Please use "Download & Update Rules" to fetch your initial ruleset, automatic updating can be scheduled after the first download.') }} diff --git a/src/opnsense/scripts/suricata/lib/downloader.py b/src/opnsense/scripts/suricata/lib/downloader.py index b960c6d22..4597c033f 100644 --- a/src/opnsense/scripts/suricata/lib/downloader.py +++ b/src/opnsense/scripts/suricata/lib/downloader.py @@ -110,7 +110,7 @@ class Downloader(object): else: return src.read() - def download(self, proto, url, url_filename, filename, input_filter): + def download(self, proto, url, url_filename, filename, input_filter, auth = None): """ download ruleset file :param proto: protocol (http,https) :param url: download url @@ -121,7 +121,13 @@ class Downloader(object): frm_url = url.replace('//', '/').replace(':/', '://') # stream to temp file if frm_url not in self._download_cache: - req = requests.get(url=frm_url, stream=True, verify=False) + req_opts = dict() + req_opts['url'] = frm_url + req_opts['stream'] = True + if auth is not None: + req_opts['auth'] = auth + req = requests.get(**req_opts) + if req.status_code == 200: src = tempfile.NamedTemporaryFile('wb+', 10240) while True: diff --git a/src/opnsense/scripts/suricata/rule-updater.py b/src/opnsense/scripts/suricata/rule-updater.py index 72ba448ed..cd4c998c8 100755 --- a/src/opnsense/scripts/suricata/rule-updater.py +++ b/src/opnsense/scripts/suricata/rule-updater.py @@ -82,5 +82,9 @@ if __name__ == '__main__': pass else: input_filter = enabled_rulefiles[rule['filename']]['filter'] + if ('username' in rule['source'] and 'password' in rule['source']): + auth = (rule['source']['username'], rule['source']['password']) + else: + auth = None dl.download(proto=download_proto, url=rule['url'], url_filename=rule['url_filename'], - filename=rule['filename'], input_filter=input_filter) + filename=rule['filename'], input_filter=input_filter, auth=auth) diff --git a/src/opnsense/service/templates/OPNsense/IDS/rule-updater.config b/src/opnsense/service/templates/OPNsense/IDS/rule-updater.config index f26813e37..51d48892f 100644 --- a/src/opnsense/service/templates/OPNsense/IDS/rule-updater.config +++ b/src/opnsense/service/templates/OPNsense/IDS/rule-updater.config @@ -2,6 +2,14 @@ create configuration for OPNsense suricata rule file downloader #} # autogenerated, do not edit. +[__properties__] +{% if helpers.exists('OPNsense.IDS.fileTags.tag') %} +{% for tag in helpers.toList('OPNsense.IDS.fileTags.tag') %} +{{tag.property}}={{tag.value}} + +{% endfor %} +{% endif %} + {% if helpers.exists('OPNsense.IDS.files.file') %} {% for file in helpers.toList('OPNsense.IDS.files.file') %} [{{file.filename|default('-')}}]