From 41fcaf2436dfee412ddb03e4ac715cb162d85cf3 Mon Sep 17 00:00:00 2001 From: Ad Schellevis Date: Sun, 9 Jun 2019 16:28:50 +0100 Subject: [PATCH] move to syslog-ng (#3524) Integrate syslog-ng into core system without removing current circular log functionality, the basic idea here is to keep syslogd to write circular log files which will turn into a consumer of syslog-ng messages. New sources, which will register automatically in s_all, are defined in /usr/local/opnsense/service/templates/OPNsense/Syslog/sources/. Plugins can write additional sources here (such as log sockets). Legacy remote log targets are collected in syslog-ng-legacy-remote.conf, which should mimic the previous syslogd targets. --- src/etc/inc/plugins.inc.d/core.inc | 11 + src/etc/inc/system.inc | 63 +----- .../templates/OPNsense/Syslog/+TARGETS | 3 + .../templates/OPNsense/Syslog/rc.conf.d | 2 + .../OPNsense/Syslog/sources/001-local.conf | 5 + .../Syslog/syslog-ng-legacy-remote.conf | 101 +++++++++ .../OPNsense/Syslog/syslog-ng-legacy.conf | 13 ++ .../templates/OPNsense/Syslog/syslog-ng.conf | 194 ++---------------- 8 files changed, 163 insertions(+), 229 deletions(-) create mode 100644 src/opnsense/service/templates/OPNsense/Syslog/rc.conf.d create mode 100644 src/opnsense/service/templates/OPNsense/Syslog/sources/001-local.conf create mode 100644 src/opnsense/service/templates/OPNsense/Syslog/syslog-ng-legacy-remote.conf create mode 100644 src/opnsense/service/templates/OPNsense/Syslog/syslog-ng-legacy.conf diff --git a/src/etc/inc/plugins.inc.d/core.inc b/src/etc/inc/plugins.inc.d/core.inc index 417dfc9b5..007f36020 100644 --- a/src/etc/inc/plugins.inc.d/core.inc +++ b/src/etc/inc/plugins.inc.d/core.inc @@ -89,6 +89,17 @@ function core_services() 'name' => 'syslog', ); + $services[] = array( + 'description' => gettext('Local Syslog-NG'), + 'mwexec' => array( + 'restart' => array('/usr/local/etc/rc.d/syslog-ng restart'), + 'start' => array('/usr/local/etc/rc.d/syslog-ng start'), + 'stop' => array('/usr/local/etc/rc.d/syslog-ng stop'), + ), + 'pidfile' => '/var/run/syslog-ng.pid', + 'name' => 'syslog-ng', + ); + return $services; } diff --git a/src/etc/inc/system.inc b/src/etc/inc/system.inc index aaa54589b..8b74d49f8 100644 --- a/src/etc/inc/system.inc +++ b/src/etc/inc/system.inc @@ -558,24 +558,6 @@ function system_syslogd_fixup_server($server) } } -function system_syslogd_get_remote_servers($syslogcfg, $facility = "*.*") -{ - // Rather than repeatedly use the same code, use this function to build a list of remote servers. - $facility .= " ". - $remote_servers = ""; - $pad_to = 56; - $padding = ceil(($pad_to - strlen($facility))/8)+1; - if (!empty($syslogcfg['remoteserver'])) { - $remote_servers .= "{$facility}" . str_repeat("\t", $padding) . "@" . system_syslogd_fixup_server($syslogcfg['remoteserver']) . "\n"; - } - if (!empty($syslogcfg['remoteserver2'])) { - $remote_servers .= "{$facility}" . str_repeat("\t", $padding) . "@" . system_syslogd_fixup_server($syslogcfg['remoteserver2']) . "\n"; - } - if (!empty($syslogcfg['remoteserver3'])) { - $remote_servers .= "{$facility}" . str_repeat("\t", $padding) . "@" . system_syslogd_fixup_server($syslogcfg['remoteserver3']) . "\n"; - } - return $remote_servers; -} function system_syslogd_extra_local($logsocket) { @@ -618,16 +600,16 @@ function system_syslogd_start($verbose = false, $restart = false) * the key as a "name" entry in the array... */ $syslogconfs['configd'] = array('facility' => array('configd.py')); - $syslogconfs['dhcpd'] = array('facility' => array('dhcpd', 'dhcrelay'), 'local' => '/var/dhcpd/var/run/log', 'remote' => 'dhcp'); - $syslogconfs['filter'] = array('facility' => array('filterlog'), 'remote' => 'filter'); - $syslogconfs['gateways'] = array('facility' => array('dpinger'), 'remote' => 'apinger'); + $syslogconfs['dhcpd'] = array('facility' => array('dhcpd', 'dhcrelay')); + $syslogconfs['filter'] = array('facility' => array('filterlog')); + $syslogconfs['gateways'] = array('facility' => array('dpinger')); $syslogconfs['lighttpd'] = array('facility' => array('lighttpd')); $syslogconfs['pkg'] = array('facility' => array('pkg', 'pkg-static')); - $syslogconfs['portalauth'] = array('facility' => array('captiveportal'), 'remote' => 'portalauth'); + $syslogconfs['portalauth'] = array('facility' => array('captiveportal')); $syslogconfs['ppps'] = array('facility' => array('ppp')); - $syslogconfs['resolver'] = array('facility' => array('unbound'), 'local' => '/var/unbound/var/run/log', 'remote' => 'dns'); + $syslogconfs['resolver'] = array('facility' => array('unbound')); $syslogconfs['routing'] = array('facility' => array('radvd', 'routed', 'rtsold', 'olsrd', 'zebra', 'ospfd', 'bgpd', 'miniupnpd')); - $syslogconfs['wireless'] = array('facility' => array('hostapd'), 'remote' => 'hostapd'); + $syslogconfs['wireless'] = array('facility' => array('hostapd')); $separatelogfacilities = array(); foreach ($syslogconfs as $logTopic => $logConfig) { @@ -636,13 +618,11 @@ function system_syslogd_start($verbose = false, $restart = false) if (!isset($syslogcfg['disablelocallogging'])) { $syslogconf .= "*.* {$log_directive}/var/log/{$logTopic}.log\n"; } - if (!empty($logConfig['remote']) && !empty($syslogcfg[$logConfig['remote']]) && !empty($syslogcfg['enable'])) { - $syslogconf .= system_syslogd_get_remote_servers($syslogcfg, "*.*"); - } if (!empty($logConfig['local'])) { $syslogd_extra .= system_syslogd_extra_local($logConfig['local']); } } + $syslogd_extra .= '-p /var/run/legacy_log -S /var/run/legacy_logpriv -k -s -s '; asort($separatelogfacilities); $facilitylist = implode(',', array_unique($separatelogfacilities)); @@ -663,34 +643,8 @@ auth.info;authpriv.info;user.* |exec /usr/local/sbin/sshlockout_pf 15 EOD; } - if (!empty($syslogcfg['enable'])) { - /* XXX most remote facilities are already sent to remote using the 'remote' keyword */ - if (isset($syslogcfg['system'])) { - $syslogconf .= system_syslogd_get_remote_servers($syslogcfg, '*.notice;kern.debug;lpr.info;mail.crit;daemon.none'); - $syslogconf .= system_syslogd_get_remote_servers($syslogcfg, 'news.err;local0.none;local3.none;local4.none'); - $syslogconf .= system_syslogd_get_remote_servers($syslogcfg, 'local7.none'); - $syslogconf .= system_syslogd_get_remote_servers($syslogcfg, 'security.*'); - $syslogconf .= system_syslogd_get_remote_servers($syslogcfg, 'auth.info;authpriv.info;daemon.info'); - $syslogconf .= system_syslogd_get_remote_servers($syslogcfg, '*.emerg'); - } - if (isset($syslogcfg['logall'])) { - // Make everything mean everything, including facilities excluded above. - $syslogconf .= "!*\n"; - $syslogconf .= system_syslogd_get_remote_servers($syslogcfg, '*.*'); - } - } file_put_contents('/var/etc/syslog.conf', $syslogconf); - - if (!empty($syslogcfg['sourceip'])) { - $ifaddr = $syslogcfg['ipproto'] == 'ipv6' ? - get_interface_ipv6($syslogcfg['sourceip']) : - get_interface_ip($syslogcfg['sourceip']); - if (is_ipaddr($ifaddr)) { - $syslogd_extra .= exec_safe('-b %s ', $ifaddr); - } - } - $syslogd_extra .= exec_safe('-f %s ', '/var/etc/syslog.conf'); // setup log files for all facilities including default @@ -707,9 +661,12 @@ EOD; if (!$restart && isvalidpid('/var/run/syslog.pid')) { killbypid('/var/run/syslog.pid', 'HUP'); + mwexecf("/usr/sbin/service syslog-ng reload"); } else { killbypid('/var/run/syslog.pid', 'TERM', true); + mwexecf("/usr/sbin/service syslog-ng stop"); mwexecf("/usr/local/sbin/syslogd -s -c -c -P %s {$syslogd_extra}", '/var/run/syslog.pid'); + mwexecf("/usr/sbin/service syslog-ng start"); } if ($verbose) { diff --git a/src/opnsense/service/templates/OPNsense/Syslog/+TARGETS b/src/opnsense/service/templates/OPNsense/Syslog/+TARGETS index b6393a255..5b29670ba 100644 --- a/src/opnsense/service/templates/OPNsense/Syslog/+TARGETS +++ b/src/opnsense/service/templates/OPNsense/Syslog/+TARGETS @@ -1,2 +1,5 @@ +rc.conf.d:/etc/rc.conf.d/syslog_ng newsyslog.conf:/etc/newsyslog.conf syslog-ng.conf:/usr/local/etc/syslog-ng.conf +syslog-ng-legacy.conf:/usr/local/etc/syslog-ng.conf.d/legacy.conf +syslog-ng-legacy-remote.conf:/usr/local/etc/syslog-ng.conf.d/legacy-remote.conf diff --git a/src/opnsense/service/templates/OPNsense/Syslog/rc.conf.d b/src/opnsense/service/templates/OPNsense/Syslog/rc.conf.d new file mode 100644 index 000000000..cf929c18f --- /dev/null +++ b/src/opnsense/service/templates/OPNsense/Syslog/rc.conf.d @@ -0,0 +1,2 @@ +syslog_ng_enable="YES" +syslog_ng_pid="/var/run/syslog-ng.pid" diff --git a/src/opnsense/service/templates/OPNsense/Syslog/sources/001-local.conf b/src/opnsense/service/templates/OPNsense/Syslog/sources/001-local.conf new file mode 100644 index 000000000..e73111766 --- /dev/null +++ b/src/opnsense/service/templates/OPNsense/Syslog/sources/001-local.conf @@ -0,0 +1,5 @@ + internal(); + unix-dgram("/var/run/log"); + unix-dgram("/var/run/logpriv" perm(0600)); + unix-dgram("/var/dhcpd/var/run/log" dir_perm(0755)); + unix-dgram("/var/unbound/var/run/log" dir_perm(0755)); diff --git a/src/opnsense/service/templates/OPNsense/Syslog/syslog-ng-legacy-remote.conf b/src/opnsense/service/templates/OPNsense/Syslog/syslog-ng-legacy-remote.conf new file mode 100644 index 000000000..b26eaed2c --- /dev/null +++ b/src/opnsense/service/templates/OPNsense/Syslog/syslog-ng-legacy-remote.conf @@ -0,0 +1,101 @@ +{% if not helpers.empty('syslog.enable') %} + +{% if syslog.ipproto == 'ipv6'%} +{% set ipprotocol = "6" %} +{% else %} +{% set ipprotocol = "4" %} +{% endif %} + +{# generate localip tag if we can find a static address #} +{% set ns = namespace() %} +{% for intf_key,intf_item in interfaces.items() %} +{% if ipprotocol == "4" %} +{% if intf_key == syslog.sourceip and intf_item.ipaddr and intf_item.ipaddr != 'dhcp' %} +{% set ns.localiptag = 'localip(' ~ intf_item.ipaddr ~ ')' %} +{% endif %} +{% else %} +{% if intf_key == syslog.sourceip and intf_item.ipaddrv6 and intf_item.ipaddrv6.count(':') > 0 %} +{% set ns.localiptag = 'localip(' ~ intf_item.ipaddrv6 ~ ')' %} +{% endif %} +{% endif %} +{% endfor %} + + +destination d_legacy_remote { + +{% for server in ['remoteserver', 'remoteserver2', 'remoteserver3'] %} +{% if not helpers.empty('syslog.' + server) %} +network("{{syslog[server]}}" transport("udp") port(514) ip-protocol({{ipprotocol}}) {{ns.localiptag}}); +{% endif %} +{% endfor %} + +}; + +{% if not helpers.empty('syslog.logall') %} +### ALL #### +log { + source(s_all); + destination(d_legacy_remote); +}; + +{% else %} + + +# section filters +filter f_remote_system { + not facility(daemon, local0, local1, local2, local3, local4, local5, local6, local7, user); +}; +filter f_remote_filter { + program(filterlog); +}; +filter f_remote_dhcp { + program("dhcrelay") or + program("dhcpd"); +}; +filter f_remote_dns { + program("unbound") or + program("dnsmasq"); +}; +filter f_remote_mail { + program("postfix"); +}; +filter f_remote_portalauth { + program("captiveportal"); +}; +filter f_remote_vpn { + program("l2tps") or + program("poes") or + program("pptps") or + program("charon") or + program("openvpn") or + program("tinc*"); +}; +filter f_remote_ids { + program("suricata"); +}; +filter f_remote_apinger { + program("dpinger"); +}; +filter f_remote_relayd { + program("haproxy") or + program("relayd"); +}; +filter f_remote_hostapd { + program("hostapd"); +}; + +{% for section in ['system', 'filter', 'dhcp', 'dns', 'mail', 'portalauth', 'vpn', 'ids', 'apinger', 'relayd', 'hostapd'] %} +{% if not helpers.empty('syslog.'+section) %} +### log section {{section}} #### +log { + source(s_all); + filter(f_remote_{{section}}); + destination(d_legacy_remote); +}; +{% endif %} +{% endfor %} + + +{% endif %} + +{% endif %} diff --git a/src/opnsense/service/templates/OPNsense/Syslog/syslog-ng-legacy.conf b/src/opnsense/service/templates/OPNsense/Syslog/syslog-ng-legacy.conf new file mode 100644 index 000000000..c89d8f917 --- /dev/null +++ b/src/opnsense/service/templates/OPNsense/Syslog/syslog-ng-legacy.conf @@ -0,0 +1,13 @@ +# +# OPNsense legacy log target +# send all received local events to platform standard syslogd +# + +destination legacy_dst { + unix-dgram("/var/run/legacy_log" template("${MSGHDR}${MESSAGE}\n")); +}; + +log { + source(s_all); + destination(legacy_dst); +}; diff --git a/src/opnsense/service/templates/OPNsense/Syslog/syslog-ng.conf b/src/opnsense/service/templates/OPNsense/Syslog/syslog-ng.conf index b854fb0ee..1608b52d6 100644 --- a/src/opnsense/service/templates/OPNsense/Syslog/syslog-ng.conf +++ b/src/opnsense/service/templates/OPNsense/Syslog/syslog-ng.conf @@ -1,184 +1,26 @@ -@version:3.20 +@version:3.21 @include "scl.conf" -# -# This sample configuration file is essentially equilivent to the stock -# FreeBSD /etc/syslog.conf file. -# -# $FreeBSD: head/sysutils/syslog-ng/files/syslog-ng.conf.sample 340872 2014-01-24 00:14:07Z mat $ -# - # # options # -options { chain_hostnames(off); flush_lines(0); threaded(yes); }; +options { + keep_hostname(yes); + chain_hostnames(off); + flush_lines(0); + threaded(yes); + create_dirs(yes); +}; -# -# sources -# -#source src { system(); udp(); internal(); }; +source s_all { +{# + merge all source configurations +#} +{% for sfilename in helpers.glob("OPNsense/Syslog/sources/*.conf") %}{% + include sfilename without context +%} +{% endfor %} -# -# destinations -# -#destination messages { file("/var/log/messages"); }; -#destination security { file("/var/log/security"); }; -#destination authlog { file("/var/log/auth.log"); }; -#destination maillog { file("/var/log/maillog"); }; -#destination lpd-errs { file("/var/log/lpd-errs"); }; -#destination xferlog { file("/var/log/xferlog"); }; -#destination cron { file("/var/log/cron"); }; -#destination debuglog { file("/var/log/debug.log"); }; -#destination consolelog { file("/var/log/console.log"); }; -#destination all { file("/var/log/all.log"); }; -#destination newscrit { file("/var/log/news/news.crit"); }; -#destination newserr { file("/var/log/news/news.err"); }; -#destination newsnotice { file("/var/log/news/news.notice"); }; -#destination slip { file("/var/log/slip.log"); }; -#destination ppp { file("/var/log/ppp.log"); }; -#destination console { file("/dev/console"); }; -#destination allusers { usertty("*"); }; -#destination loghost { udp("loghost" port(514)); }; +}; -# -# log facility filters -# -filter f_auth { facility(auth); }; -filter f_authpriv { facility(authpriv); }; -filter f_not_authpriv { not facility(authpriv); }; -#filter f_console { facility(console); }; -filter f_cron { facility(cron); }; -filter f_daemon { facility(daemon); }; -filter f_ftp { facility(ftp); }; -filter f_kern { facility(kern); }; -filter f_lpr { facility(lpr); }; -filter f_mail { facility(mail); }; -filter f_news { facility(news); }; -filter f_security { facility(security); }; -filter f_user { facility(user); }; -filter f_uucp { facility(uucp); }; -filter f_local0 { facility(local0); }; -filter f_local1 { facility(local1); }; -filter f_local2 { facility(local2); }; -filter f_local3 { facility(local3); }; -filter f_local4 { facility(local4); }; -filter f_local5 { facility(local5); }; -filter f_local6 { facility(local6); }; -filter f_local7 { facility(local7); }; - -# -# log level filters -# -filter f_emerg { level(emerg); }; -filter f_alert { level(alert..emerg); }; -filter f_crit { level(crit..emerg); }; -filter f_err { level(err..emerg); }; -filter f_warning { level(warning..emerg); }; -filter f_notice { level(notice..emerg); }; -filter f_info { level(info..emerg); }; -filter f_debug { level(debug..emerg); }; -filter f_is_debug { level(debug); }; - -# -# program filters -# -filter f_ppp { program("ppp"); }; -filter f_slip { program("startslip"); }; - -# -# *.err;kern.warning;auth.notice;mail.crit /dev/console -# -#log { source(src); filter(f_err); destination(console); }; -#log { source(src); filter(f_kern); filter(f_warning); destination(console); }; -#log { source(src); filter(f_auth); filter(f_notice); destination(console); }; -#log { source(src); filter(f_mail); filter(f_crit); destination(console); }; - -# -# *.notice;authpriv.none;kern.debug;lpr.info;mail.crit;news.err /var/log/messages -# -#log { source(src); filter(f_notice); filter(f_not_authpriv); destination(messages); }; -#log { source(src); filter(f_kern); filter(f_debug); destination(messages); }; -#log { source(src); filter(f_lpr); filter(f_info); destination(messages); }; -#log { source(src); filter(f_mail); filter(f_crit); destination(messages); }; -#log { source(src); filter(f_news); filter(f_err); destination(messages); }; - -# -# security.* /var/log/security -# -#log { source(src); filter(f_security); destination(security); }; - -# -# auth.info;authpriv.info /var/log/auth.log -#log { source(src); filter(f_auth); filter(f_info); destination(authlog); }; -#log { source(src); filter(f_authpriv); filter(f_info); destination(authlog); }; - -# -# mail.info /var/log/maillog -# -#log { source(src); filter(f_mail); filter(f_info); destination(maillog); }; - -# -# lpr.info /var/log/lpd-errs -# -#log { source(src); filter(f_lpr); filter(f_info); destination(lpd-errs); }; - -# -# ftp.info /var/log/xferlog -# -#log { source(src); filter(f_ftp); filter(f_info); destination(xferlog); }; - -# -# cron.* /var/log/cron -# -#log { source(src); filter(f_cron); destination(cron); }; - -# -# *.=debug /var/log/debug.log -# -#log { source(src); filter(f_is_debug); destination(debuglog); }; - -# -# *.emerg * -# -#log { source(src); filter(f_emerg); destination(allusers); }; - -# -# uncomment this to log all writes to /dev/console to /var/log/console.log -# console.info /var/log/console.log -# -#log { source(src); filter(f_console); filter(f_info); destination(consolelog); }; - -# -# uncomment this to enable logging of all log messages to /var/log/all.log -# touch /var/log/all.log and chmod it to mode 600 before it will work -# *.* /var/log/all.log -# -#log { source(src); destination(all); }; - -# -# uncomment this to enable logging to a remote loghost named loghost -# *.* @loghost -# -#log { source(src); destination(loghost); }; - -# -# uncomment these if you're running inn -# news.crit /var/log/news/news.crit -# news.err /var/log/news/news.err -# news.notice /var/log/news/news.notice -# -#log { source(src); filter(f_news); filter(f_crit); destination(newscrit); }; -#log { source(src); filter(f_news); filter(f_err); destination(newserr); }; -#log { source(src); filter(f_news); filter(f_notice); destination(newsnotice); }; - -# -# !startslip -# *.* /var/log/slip.log -# -#log { source(src); filter(f_slip); destination(slip); }; - -# -# !ppp -# *.* /var/log/ppp.log -# -#log { source(src); filter(f_ppp); destination(ppp); }; +@include "/usr/local/etc/syslog-ng.conf.d/*.conf"