lucaspalomodevelop/core
1
0
Fork 0
mirror of https://github.com/lucaspalomodevelop/core.git synced 2026-03-15 00:54:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Web proxy: add json output, following Elastic Common Schema (ECS) reference. closes https://github.com/opnsense/core/issues/4244

Browse Source 
o Extracts most of the attributes from our extended log format, when X-Request-Event-Id header is set it will be included as event id.
o Added log format for internal ui parsing (extract timestamp)

* https://github.com/elastic/ecs/blob/master/generated/csv/fields.csv
* http://www.squid-cache.org/Doc/config/logformat/

Sponsored by Incenter Technology (https://www.incenter.tech/)
This commit is contained in:
Ad Schellevis 2020-08-07 16:53:18 +02:00
parent 2344a7b40c
commit 5bd793a8a0
3 changed files with 51 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/opnsense/mvc/app/models/OPNsense/Proxy/Proxy.xml
View File

@ -45,7 +45,9 @@
<BlankDesc>File</BlankDesc>
<OptionValues>
<file_extendend>File (Extended)</file_extendend>
<file_json>File (Json)</file_json>
<syslog>Syslog</syslog>
<syslog_json>Syslog (Json)</syslog_json>
</OptionValues>
</target>
</logging>

40
src/opnsense/scripts/systemhealth/logformats/squid.py
View File

@ -32,7 +32,7 @@ squid_timeformat = r'^(\d{4}/\d{1,2}/\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}).*'
class SquidLogFormat(BaseLogFormat):
def __init__(self, filename):
super(SquidLogFormat, self).__init__(filename)
super().__init__(filename)
self._priority = 100
def match(self, line):
@ -44,6 +44,10 @@ class SquidLogFormat(BaseLogFormat):
grp = tmp.group(1)
return datetime.datetime.strptime(grp, "%Y/%m/%d %H:%M:%S").isoformat()
@staticmethod
def process_name(line):
return "squid"
@staticmethod
def line(line):
return line[19:].strip()
@ -51,7 +55,7 @@ class SquidLogFormat(BaseLogFormat):
class SquidExtLogFormat(BaseLogFormat):
def __init__(self, filename):
super(SquidExtLogFormat, self).__init__(filename)
super().__init__(filename)
self._priority = 120
def match(self, line):
@ -63,8 +67,40 @@ class SquidExtLogFormat(BaseLogFormat):
grp = tmp.group(1)
return datetime.datetime.strptime(grp[1:].split()[0], "%d/%b/%Y:%H:%M:%S").isoformat()
@staticmethod
def process_name(line):
return "squid"
@staticmethod
def line(line):
tmp = re.match(squid_ext_timeformat, line)
grp = tmp.group(1)
return line.replace(grp, '')
class SquidJsonLogFormat(BaseLogFormat):
def __init__(self, filename):
super().__init__(filename)
self._priority = 140
local_now = datetime.datetime.now()
utc_now = datetime.datetime.utcnow()
self._localtimezone = datetime.timezone(local_now - utc_now)
def match(self, line):
return self._filename.find('squid') > -1 and line.find('"@timestamp"') > -1
def timestamp(self, line):
tmp = line[line.find('"@timestamp"')+13:].split(',')[0].strip().strip('"')
try:
return datetime.datetime.strptime(tmp, "%Y-%m-%dT%H:%M:%S%z")\
.astimezone(self._localtimezone).isoformat().split('.')[0].split('+')[0]
except ValueError:
return None
@staticmethod
def process_name(line):
return "squid"
@staticmethod
def line(line):
return line

14
src/opnsense/service/templates/OPNsense/Proxy/squid.conf
View File

@ -367,12 +367,20 @@ access_log none
acl accesslog_ignore src {{ OPNsense.proxy.general.logging.ignoreLogACL.replace(',', ' ') }}
{% endif %}
{% if OPNsense.proxy.general.logging.target|default('') == 'syslog' %}
access_log syslog:local4.info {% if OPNsense.proxy.general.logging.ignoreLogACL|default('') %}!accesslog_ignore {% endif %}
access_log syslog:local4.info {% if not helpers.empty('OPNsense.proxy.general.logging.ignoreLogACL') %}!accesslog_ignore {% endif %}
{% elif OPNsense.proxy.general.logging.target|default('') == 'file_extendend' %}
logformat opnsense %>a %[ui %>eui %[un [%tl] "%rm %ru HTTP/%rv" %>Hs %<st "%{Referer}>h" "%{User-Agent}>h" %Ss:%Sh
access_log stdio:/var/log/squid/access.log opnsense {% if OPNsense.proxy.general.logging.ignoreLogACL|default('') %}!accesslog_ignore {% endif %}
access_log stdio:/var/log/squid/access.log opnsense {% if not helpers.empty('OPNsense.proxy.general.logging.ignoreLogACL') %}!accesslog_ignore {% endif %}
{% elif OPNsense.proxy.general.logging.target|default('') in ('file_json', 'syslog_json') %}
logformat opnsense {% raw %} {"@timestamp":"%{%Y-%m-%dT%H:%M:%S%z}tg","ecs":{"version":"1.0.0"},"event":{"id":"%{X-Request-Event-Id}>ha","dataset":"squid.access","duration":"%tr"},"http":{"version":"%rv","request":{"method":"%rm","referrer":"%{Referer}>h"},"response":{"body":{"bytes": %<st, "status_code": %>Hs}}},"host":{"hostname":"%>A"},"service":{"name":"proxy","type":"squid"},"source":{"ip":"%>a"},"url":{"original":"%ru"},"user":{"name":"%un"},"user_agent":{"original":"%{User-Agent}>h"},"labels":{"request_status":"%Ss","hierarchy_status":"%Sh"},"message":"%rm %ru HTTP/%rv"} {% endraw %}
{% if OPNsense.proxy.general.logging.target == 'file_json'%}
access_log stdio:/var/log/squid/access.log opnsense {% if not helpers.empty('OPNsense.proxy.general.logging.ignoreLogACL') %}!accesslog_ignore {% endif %}
{% else %}
access_log syslog:local4.info opnsense {% if not helpers.empty('OPNsense.proxy.general.logging.ignoreLogACL') %}!accesslog_ignore {% endif %}
{% endif %}
{% else %}
access_log stdio:/var/log/squid/access.log squid {% if OPNsense.proxy.general.logging.ignoreLogACL|default('') %}!accesslog_ignore {% endif %}
access_log stdio:/var/log/squid/access.log squid {% if not helpers.empty('OPNsense.proxy.general.logging.ignoreLogACL') %}!accesslog_ignore {% endif %}
{% endif %}
{% endif %}
{% endif %}