From 5bd793a8a0d42c100184cfeff62adc485063bb2d Mon Sep 17 00:00:00 2001 From: Ad Schellevis Date: Fri, 7 Aug 2020 16:53:18 +0200 Subject: [PATCH] Web proxy: add json output, following Elastic Common Schema (ECS) reference. closes https://github.com/opnsense/core/issues/4244 o Extracts most of the attributes from our extended log format, when X-Request-Event-Id header is set it will be included as event id. o Added log format for internal ui parsing (extract timestamp) * https://github.com/elastic/ecs/blob/master/generated/csv/fields.csv * http://www.squid-cache.org/Doc/config/logformat/ Sponsored by Incenter Technology (https://www.incenter.tech/) --- .../mvc/app/models/OPNsense/Proxy/Proxy.xml | 2 + .../scripts/systemhealth/logformats/squid.py | 40 ++++++++++++++++++- .../templates/OPNsense/Proxy/squid.conf | 14 +++++-- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/src/opnsense/mvc/app/models/OPNsense/Proxy/Proxy.xml b/src/opnsense/mvc/app/models/OPNsense/Proxy/Proxy.xml index 21e325929..0ddd6d168 100644 --- a/src/opnsense/mvc/app/models/OPNsense/Proxy/Proxy.xml +++ b/src/opnsense/mvc/app/models/OPNsense/Proxy/Proxy.xml @@ -45,7 +45,9 @@ File File (Extended) + File (Json) Syslog + Syslog (Json) diff --git a/src/opnsense/scripts/systemhealth/logformats/squid.py b/src/opnsense/scripts/systemhealth/logformats/squid.py index 4f983ce80..2541c024d 100755 --- a/src/opnsense/scripts/systemhealth/logformats/squid.py +++ b/src/opnsense/scripts/systemhealth/logformats/squid.py @@ -32,7 +32,7 @@ squid_timeformat = r'^(\d{4}/\d{1,2}/\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}).*' class SquidLogFormat(BaseLogFormat): def __init__(self, filename): - super(SquidLogFormat, self).__init__(filename) + super().__init__(filename) self._priority = 100 def match(self, line): @@ -44,6 +44,10 @@ class SquidLogFormat(BaseLogFormat): grp = tmp.group(1) return datetime.datetime.strptime(grp, "%Y/%m/%d %H:%M:%S").isoformat() + @staticmethod + def process_name(line): + return "squid" + @staticmethod def line(line): return line[19:].strip() @@ -51,7 +55,7 @@ class SquidLogFormat(BaseLogFormat): class SquidExtLogFormat(BaseLogFormat): def __init__(self, filename): - super(SquidExtLogFormat, self).__init__(filename) + super().__init__(filename) self._priority = 120 def match(self, line): @@ -63,8 +67,40 @@ class SquidExtLogFormat(BaseLogFormat): grp = tmp.group(1) return datetime.datetime.strptime(grp[1:].split()[0], "%d/%b/%Y:%H:%M:%S").isoformat() + @staticmethod + def process_name(line): + return "squid" + @staticmethod def line(line): tmp = re.match(squid_ext_timeformat, line) grp = tmp.group(1) return line.replace(grp, '') + + +class SquidJsonLogFormat(BaseLogFormat): + def __init__(self, filename): + super().__init__(filename) + self._priority = 140 + local_now = datetime.datetime.now() + utc_now = datetime.datetime.utcnow() + self._localtimezone = datetime.timezone(local_now - utc_now) + + def match(self, line): + return self._filename.find('squid') > -1 and line.find('"@timestamp"') > -1 + + def timestamp(self, line): + tmp = line[line.find('"@timestamp"')+13:].split(',')[0].strip().strip('"') + try: + return datetime.datetime.strptime(tmp, "%Y-%m-%dT%H:%M:%S%z")\ + .astimezone(self._localtimezone).isoformat().split('.')[0].split('+')[0] + except ValueError: + return None + + @staticmethod + def process_name(line): + return "squid" + + @staticmethod + def line(line): + return line diff --git a/src/opnsense/service/templates/OPNsense/Proxy/squid.conf b/src/opnsense/service/templates/OPNsense/Proxy/squid.conf index e47c906df..9031c546c 100644 --- a/src/opnsense/service/templates/OPNsense/Proxy/squid.conf +++ b/src/opnsense/service/templates/OPNsense/Proxy/squid.conf @@ -367,12 +367,20 @@ access_log none acl accesslog_ignore src {{ OPNsense.proxy.general.logging.ignoreLogACL.replace(',', ' ') }} {% endif %} {% if OPNsense.proxy.general.logging.target|default('') == 'syslog' %} -access_log syslog:local4.info {% if OPNsense.proxy.general.logging.ignoreLogACL|default('') %}!accesslog_ignore {% endif %} +access_log syslog:local4.info {% if not helpers.empty('OPNsense.proxy.general.logging.ignoreLogACL') %}!accesslog_ignore {% endif %} {% elif OPNsense.proxy.general.logging.target|default('') == 'file_extendend' %} logformat opnsense %>a %[ui %>eui %[un [%tl] "%rm %ru HTTP/%rv" %>Hs %h" "%{User-Agent}>h" %Ss:%Sh -access_log stdio:/var/log/squid/access.log opnsense {% if OPNsense.proxy.general.logging.ignoreLogACL|default('') %}!accesslog_ignore {% endif %} +access_log stdio:/var/log/squid/access.log opnsense {% if not helpers.empty('OPNsense.proxy.general.logging.ignoreLogACL') %}!accesslog_ignore {% endif %} +{% elif OPNsense.proxy.general.logging.target|default('') in ('file_json', 'syslog_json') %} +logformat opnsense {% raw %} {"@timestamp":"%{%Y-%m-%dT%H:%M:%S%z}tg","ecs":{"version":"1.0.0"},"event":{"id":"%{X-Request-Event-Id}>ha","dataset":"squid.access","duration":"%tr"},"http":{"version":"%rv","request":{"method":"%rm","referrer":"%{Referer}>h"},"response":{"body":{"bytes": %Hs}}},"host":{"hostname":"%>A"},"service":{"name":"proxy","type":"squid"},"source":{"ip":"%>a"},"url":{"original":"%ru"},"user":{"name":"%un"},"user_agent":{"original":"%{User-Agent}>h"},"labels":{"request_status":"%Ss","hierarchy_status":"%Sh"},"message":"%rm %ru HTTP/%rv"} {% endraw %} + +{% if OPNsense.proxy.general.logging.target == 'file_json'%} +access_log stdio:/var/log/squid/access.log opnsense {% if not helpers.empty('OPNsense.proxy.general.logging.ignoreLogACL') %}!accesslog_ignore {% endif %} +{% else %} +access_log syslog:local4.info opnsense {% if not helpers.empty('OPNsense.proxy.general.logging.ignoreLogACL') %}!accesslog_ignore {% endif %} +{% endif %} {% else %} -access_log stdio:/var/log/squid/access.log squid {% if OPNsense.proxy.general.logging.ignoreLogACL|default('') %}!accesslog_ignore {% endif %} +access_log stdio:/var/log/squid/access.log squid {% if not helpers.empty('OPNsense.proxy.general.logging.ignoreLogACL') %}!accesslog_ignore {% endif %} {% endif %} {% endif %} {% endif %}