lucaspalomodevelop/core
1
0
Fork 0
mirror of https://github.com/lucaspalomodevelop/core.git synced 2026-03-16 09:34:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

python 2-->3 / configd

Browse Source 
First (functional) attempt, this needs to stay on devel for some time there might be dragons ;)

src/etc/rc.d/configd --> command_interpreter could cause restart issues after an upgrade, the rc system doesn't like command changes it seems. Maybe not a real world problem, just haven't tried it yet.
unit tests are somewhat functional, although generating all templates will likely fail, since the test config doesn't include all data involved.
This commit is contained in:
Ad Schellevis 2019-02-22 21:03:42 +01:00
parent 47a3b2419d
commit 91be9a6974
13 changed files with 83 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/etc/rc.d/configd
View File

@ -26,7 +26,7 @@ configd_load_rc_config()
required_files=""
command_args="${required_args}"
command=/usr/local/opnsense/service/configd.py
command_interpreter=/usr/local/bin/python2.7
command_interpreter=/usr/local/bin/python3.6
}
#

4
src/opnsense/service/configd.py
View File

@ -1,8 +1,8 @@
#!/usr/local/bin/python2.7
#!/usr/local/bin/python3.6
# -*- coding: utf-8 -*-
"""
Copyright (c) 2014-2016 Ad Schellevis <ad@opnsense.org>
Copyright (c) 2014-2019 Ad Schellevis <ad@opnsense.org>
All rights reserved.
Redistribution and use in source and binary forms, with or without

12
src/opnsense/service/configd_ctl.py
View File

@ -1,7 +1,7 @@
#!/usr/local/bin/python2.7
#!/usr/local/bin/python3.6
"""
Copyright (c) 2015 Ad Schellevis <ad@opnsense.org>
Copyright (c) 2015-2019 Ad Schellevis <ad@opnsense.org>
All rights reserved.
Redistribution and use in source and binary forms, with or without
@ -59,10 +59,10 @@ def exec_config_cmd(exec_command):
return None
try:
sock.send(exec_command)
sock.send(exec_command.encode())
data = []
while True:
line = sock.recv(65536)
line = sock.recv(65536).decode()
if line:
data.append(line)
else:
@ -82,7 +82,7 @@ socket.setdefaulttimeout(120)
# validate parameters
if len(sys.argv) <= 1:
print ('usage : %s [-m] <command>'%sys.argv[0])
print('usage : %s [-m] <command>'%sys.argv[0])
sys.exit(0)
# check if configd socket exists
@ -95,7 +95,7 @@ while not os.path.exists(configd_socket_name):
i += 1
if not os.path.exists(configd_socket_name):
print ('configd socket missing (@%s)'%configd_socket_name)
print('configd socket missing (@%s)'%configd_socket_name)
sys.exit(-1)
if sys.argv[1] == '-m':

2
src/opnsense/service/modules/addons/template_helpers.py
View File

@ -1,5 +1,5 @@
"""
Copyright (c) 2015 Ad Schellevis <ad@opnsense.org>
Copyright (c) 2015-2019 Ad Schellevis <ad@opnsense.org>
All rights reserved.
Redistribution and use in source and binary forms, with or without

2
src/opnsense/service/modules/config.py
View File

@ -1,5 +1,5 @@
"""
Copyright (c) 2015 Ad Schellevis <ad@opnsense.org>
Copyright (c) 2015-2019 Ad Schellevis <ad@opnsense.org>
All rights reserved.
Redistribution and use in source and binary forms, with or without

4
src/opnsense/service/modules/csconfigparser.py
View File

@ -1,5 +1,5 @@
"""
Copyright (c) 2015 Ad Schellevis <ad@opnsense.org>
Copyright (c) 2015-2019 Ad Schellevis <ad@opnsense.org>
All rights reserved.
Redistribution and use in source and binary forms, with or without
@ -29,7 +29,7 @@
function: make standard config parser case sensitive
"""
from ConfigParser import ConfigParser
from configparser import ConfigParser
class CSConfigParser(ConfigParser):

10
src/opnsense/service/modules/ph_inline_actions.py
View File

@ -1,5 +1,5 @@
"""
Copyright (c) 2014 Ad Schellevis <ad@opnsense.org>
Copyright (c) 2014-2019 Ad Schellevis <ad@opnsense.org>
All rights reserved.
Redistribution and use in source and binary forms, with or without
@ -30,8 +30,8 @@
"""
import syslog
import template
import config
from . import template
from . import config
__author__ = 'Ad Schellevis'
@ -86,11 +86,11 @@ def execute(action, parameters):
return 'ERR'
elif action.command == 'configd.actions':
# list all available configd actions
from processhandler import ActionHandler
from .processhandler import ActionHandler
act_handler = ActionHandler()
actions = act_handler.list_actions(['message', 'description'])
if unicode(parameters).lower() == 'json':
if str(parameters).lower() == 'json':
import json
return json.dumps(actions)
else:

34
src/opnsense/service/modules/processhandler.py
View File

@ -1,5 +1,5 @@
"""
Copyright (c) 2014 Ad Schellevis <ad@opnsense.org>
Copyright (c) 2014-2019 Ad Schellevis <ad@opnsense.org>
All rights reserved.
Redistribution and use in source and binary forms, with or without
@ -34,13 +34,13 @@ import socket
import traceback
import syslog
import threading
import ConfigParser
import configparser
import glob
import time
import uuid
import shlex
import tempfile
import ph_inline_actions
from . import ph_inline_actions
from modules import singleton
__author__ = 'Ad Schellevis'
@ -126,7 +126,7 @@ class Handler(object):
return
except Exception:
# something went wrong... send traceback to syslog, restart listener (wait for a short time)
print (traceback.format_exc())
print(traceback.format_exc())
syslog.syslog(syslog.LOG_ERR, 'Handler died on %s' % traceback.format_exc())
time.sleep(1)
@ -163,12 +163,12 @@ class HandlerClient(threading.Thread):
# noinspection PyBroadException
try:
# receive command, maximum data length is 4k... longer messages will be truncated
data = self.connection.recv(4096)
data = self.connection.recv(4096).decode()
# map command to action
data_parts = shlex.split(data)
if len(data_parts) == 0 or len(data_parts[0]) == 0:
# no data found
self.connection.sendall('no data\n')
self.connection.sendall(('no data\n').encode())
else:
exec_command = data_parts[0]
if exec_command[0] == "&":
@ -187,7 +187,7 @@ class HandlerClient(threading.Thread):
# when running in background, return this message uuid and detach socket
if exec_in_background:
result = self.message_uuid
self.connection.sendall('%s\n%c%c%c' % (result, chr(0), chr(0), chr(0)))
self.connection.sendall(('%s\n%c%c%c' % (result, chr(0), chr(0), chr(0))).encode())
self.connection.shutdown(socket.SHUT_RDWR)
self.connection.close()
@ -200,22 +200,22 @@ class HandlerClient(threading.Thread):
if not exec_in_background:
# send response back to client( including trailing enter )
self.connection.sendall('%s\n' % result)
self.connection.sendall(('%s\n' % result).encode())
else:
# log response
syslog.syslog(syslog.LOG_INFO, "message %s [%s.%s] returned %s " % (self.message_uuid,
exec_command,
exec_action,
unicode(result)[:100]))
result[:100]))
# send end of stream characters
if not exec_in_background:
self.connection.sendall("%c%c%c" % (chr(0), chr(0), chr(0)))
self.connection.sendall(("%c%c%c" % (chr(0), chr(0), chr(0))).encode())
except SystemExit:
# ignore system exit related errors
pass
except Exception:
print (traceback.format_exc())
print(traceback.format_exc())
syslog.syslog(
syslog.LOG_ERR,
'unable to sendback response [%s] for [%s][%s][%s] {%s}, message was %s' % (result,
@ -268,7 +268,7 @@ class ActionHandler(object):
self.action_map[topic_name] = {}
# traverse config directory and open all filenames starting with actions_
cnf = ConfigParser.RawConfigParser()
cnf = configparser.RawConfigParser()
cnf.read(config_filename)
for section in cnf.sections():
# map configuration data on object
@ -370,10 +370,10 @@ class ActionHandler(object):
:return: None
"""
action_obj = self.find_action(command, action, parameters)
print ('---------------------------------------------------------------------')
print ('execute %s.%s with parameters : %s ' % (command, action, parameters))
print ('action object %s (%s) %s' % (action_obj, action_obj.command, message_uuid))
print ('---------------------------------------------------------------------')
print('---------------------------------------------------------------------')
print('execute %s.%s with parameters : %s ' % (command, action, parameters))
print('action object %s (%s) %s' % (action_obj, action_obj.command, message_uuid))
print('---------------------------------------------------------------------')
class Action(object):
@ -488,7 +488,7 @@ class Action(object):
'[%s] Script action stderr returned "%s"' %
(message_uuid, script_error_output.strip()[:255])
)
return script_output
return script_output.decode()
except Exception as script_exception:
syslog.syslog(syslog.LOG_ERR, '[%s] Script action failed with %s at %s' % (message_uuid,
script_exception,

56
src/opnsense/service/modules/template.py
View File

@ -1,5 +1,5 @@
"""
Copyright (c) 2015 Ad Schellevis <ad@opnsense.org>
Copyright (c) 2015-2019 Ad Schellevis <ad@opnsense.org>
All rights reserved.
Redistribution and use in source and binary forms, with or without
@ -39,7 +39,7 @@ import traceback
import copy
import codecs
import jinja2
import addons.template_helpers
from .addons import template_helpers
__author__ = 'Ad Schellevis'
@ -59,7 +59,6 @@ class Template(object):
self._template_dir = os.path.dirname(os.path.abspath(__file__)) + '/../templates/'
self._j2_env = jinja2.Environment(loader=jinja2.FileSystemLoader(self._template_dir), trim_blocks=True,
extensions=["jinja2.ext.do", "jinja2.ext.loopcontrols"])
# register additional filters
self._j2_env.filters['decode_idna'] = lambda x:x.decode('idna')
self._j2_env.filters['encode_idna'] = self._encode_idna
@ -68,7 +67,7 @@ class Template(object):
def _encode_idna(x):
""" encode string to idna, preserve leading dots
"""
return ''.join(map(lambda x:'.', range(len(x) - len(x.lstrip('.'))))) + x.lstrip('.').encode('idna')
return b''.join([b''.join([b'.' for x in range(len(x) - len(x.lstrip('.')))]), x.lstrip('.').encode('idna')])
def list_module(self, module_name):
""" list single module content
@ -84,20 +83,21 @@ class Template(object):
for target_source in target_sources:
if os.path.exists(target_source):
for line in open(target_source, 'r').read().split('\n'):
parts = line.split(':')
if len(parts) > 1 and parts[0].strip()[0] != '#':
source_file = parts[0].strip()
target_name = parts[1].strip()
if target_name in result['+TARGETS'].values():
syslog.syslog(syslog.LOG_NOTICE, "template overlay %s with %s" % (
target_name, os.path.basename(target_source)
))
result['+TARGETS'][source_file] = target_name
if len(parts) == 2:
result['+CLEANUP_TARGETS'][source_file] = target_name
elif parts[2].strip() != "":
result['+CLEANUP_TARGETS'][source_file] = parts[2].strip()
with open(target_source, 'r') as fhandle:
for line in fhandle.read().split('\n'):
parts = line.split(':')
if len(parts) > 1 and parts[0].strip()[0] != '#':
source_file = parts[0].strip()
target_name = parts[1].strip()
if target_name in list(result['+TARGETS'].values()):
syslog.syslog(syslog.LOG_NOTICE, "template overlay %s with %s" % (
target_name, os.path.basename(target_source)
))
result['+TARGETS'][source_file] = target_name
if len(parts) == 2:
result['+CLEANUP_TARGETS'][source_file] = target_name
elif parts[2].strip() != "":
result['+CLEANUP_TARGETS'][source_file] = parts[2].strip()
return result
def list_modules(self):
@ -155,9 +155,9 @@ class Template(object):
config_ptr = config_ptr[xmlNodeName]
elif xmlNodeName == '%':
if type(config_ptr) in (collections.OrderedDict, dict):
target_keys = config_ptr.keys()
target_keys = list(config_ptr)
else:
target_keys = map(lambda x: str(x), range(len(config_ptr)))
target_keys = [str(x) for x in range(len(config_ptr))]
else:
# config pointer is reused when the match is exact, so we need to reset it here
# if the tag was not found.
@ -217,15 +217,15 @@ class Template(object):
"""
result = []
module_data = self.list_module(module_name)
for src_template in module_data['+TARGETS'].keys():
for src_template in list(module_data['+TARGETS']):
target = module_data['+TARGETS'][src_template]
target_filename_tags = self.__find_string_tags(target)
target_filters = self.__find_filters(target_filename_tags)
result_filenames = {target: {}}
for target_filter in target_filters.keys():
for key in target_filters[target_filter].keys():
for filename in result_filenames.keys():
for target_filter in list(target_filters):
for key in list(target_filters[target_filter]):
for filename in list(result_filenames):
if target_filters[target_filter][key] is not None \
and filename.find('[%s]' % target_filter) > -1:
new_filename = filename.replace('[%s]' % target_filter, target_filters[target_filter][key])
@ -240,14 +240,14 @@ class Template(object):
except jinja2.exceptions.TemplateSyntaxError as templExc:
raise Exception("%s %s %s" % (module_name, template_filename, templExc))
for filename in result_filenames.keys():
for filename in list(result_filenames):
if not (filename.find('[') != -1 and filename.find(']') != -1):
# copy config data
cnf_data = copy.deepcopy(self._config)
cnf_data['TARGET_FILTERS'] = result_filenames[filename]
# link template helpers
self._j2_env.globals['helpers'] = addons.template_helpers.Helpers(cnf_data)
self._j2_env.globals['helpers'] = template_helpers.Helpers(cnf_data)
# make sure we're only rendering output once
if filename not in result:
@ -271,7 +271,7 @@ class Template(object):
# It looks like Jinja sometimes isn't consistent on placing this last end-of-line in.
if len(content) > 1 and content[-1] != '\n':
src_file = '%s%s' % (self._template_dir, template_filename)
src_file_handle = open(src_file, 'r')
src_file_handle = open(src_file, 'rb')
src_file_handle.seek(-1, os.SEEK_END)
last_bytes_template = src_file_handle.read()
src_file_handle.close()
@ -342,7 +342,7 @@ class Template(object):
for template_name in self.iter_modules(module_name):
syslog.syslog(syslog.LOG_NOTICE, "cleanup template container %s" % template_name)
module_data = self.list_module(module_name)
for src_template in module_data['+CLEANUP_TARGETS'].keys():
for src_template in list(module_data['+CLEANUP_TARGETS']):
target = module_data['+CLEANUP_TARGETS'][src_template]
for filename in glob.glob(target):
os.remove(filename)

2
src/opnsense/service/run_unittests.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3.6
"""
Copyright (c) 2016 Ad Schellevis <ad@opnsense.org>
All rights reserved.

1
src/opnsense/service/tests/config/config.xml
View File

@ -1,5 +1,6 @@
<?xml version="1.0"?>
<opnsense>
<version>1</version>
<interfaces>
<wan>
<enable/>

11
src/opnsense/service/tests/core.py
View File

@ -57,14 +57,14 @@ class DummySocket(object):
:param size:
:return:
"""
return self._send_data
return self._send_data.encode()
def sendall(self, data):
""" send back to "client"
:param data: text
:return:
"""
self._receive_data.append(data)
self._receive_data.append(data.decode())
def close(self):
""" close connection
@ -78,6 +78,9 @@ class DummySocket(object):
"""
return ''.join(self._receive_data)
def shutdown(self, mode):
pass
class TestCoreMethods(unittest.TestCase):
def setUp(self):
@ -106,7 +109,7 @@ class TestCoreMethods(unittest.TestCase):
action_handler=self.act_handler,
simulation_mode=False)
cmd_thread.run()
self.assertEquals(self.dummysock.getReceived()[-4:], '\n%c%c%c' % (chr(0), chr(0), chr(0)), "Invalid sequence")
self.assertEqual(self.dummysock.getReceived()[-4:], '\n%c%c%c' % (chr(0), chr(0), chr(0)), "Invalid sequence")
def test_command_unknown(self):
""" test invalid command
@ -118,7 +121,7 @@ class TestCoreMethods(unittest.TestCase):
action_handler=self.act_handler,
simulation_mode=False)
cmd_thread.run()
self.assertEquals(self.dummysock.getReceived().split('\n')[0], 'Action not found', 'Invalid response')
self.assertEqual(self.dummysock.getReceived().split('\n')[0], 'Action not found', 'Invalid response')
def test_configd_actions(self):
""" request configd command list

19
src/opnsense/service/tests/template.py
View File

@ -52,7 +52,7 @@ class TestConfigMethods(unittest.TestCase):
""" test correct config type
:return:
"""
self.assertEquals(type(self.conf.get()), collections.OrderedDict)
self.assertEqual(type(self.conf.get()), collections.OrderedDict)
def test_interface(self):
""" test existence of interface
@ -93,8 +93,8 @@ class TestTemplateMethods(unittest.TestCase):
""" test sample template
:return:
"""
generated_filenames = self.tmpl.generate('OPNsense.Sample')
self.assertEquals(len(generated_filenames), 3, 'number of output files not 3')
generated_filenames = self.tmpl.generate('OPNsense/Sample')
self.assertEqual(len(generated_filenames), 4, 'number of output files not 4')
def test_all(self):
""" Test if all expected templates are created, can only find test for static defined cases.
@ -109,11 +109,14 @@ class TestTemplateMethods(unittest.TestCase):
for filenm in files:
if filenm == '+TARGETS':
filename = '%s/%s' % (root, filenm)
for line in open(filename).read().split('\n'):
line = line.strip()
if len(line) > 1 and line[0] != '#' and line.find('[') == -1:
expected_filename = ('%s%s' % (self.output_path, line.split(':')[-1])).replace('//', '/')
self.expected_filenames[expected_filename] = {'src': filename}
with open(filename) as fhandle:
for line in fhandle.read().split('\n'):
line = line.strip()
if len(line) > 1 and line[0] != '#' and line.find('[') == -1:
expected_filename = (
'%s%s' % (self.output_path, line.split(':')[-1])
).replace('//', '/')
self.expected_filenames[expected_filename] = {'src': filename}
for filename in self.tmpl.generate('*'):
self.generated_filenames.append(filename.replace('//', '/'))