diff --git a/src/etc/rc.d/configd b/src/etc/rc.d/configd index ada7d8d3c..28ae0c557 100755 --- a/src/etc/rc.d/configd +++ b/src/etc/rc.d/configd @@ -26,7 +26,7 @@ configd_load_rc_config() required_files="" command_args="${required_args}" command=/usr/local/opnsense/service/configd.py - command_interpreter=/usr/local/bin/python2.7 + command_interpreter=/usr/local/bin/python3.6 } # diff --git a/src/opnsense/service/configd.py b/src/opnsense/service/configd.py index fb3386e3f..033321c19 100755 --- a/src/opnsense/service/configd.py +++ b/src/opnsense/service/configd.py @@ -1,8 +1,8 @@ -#!/usr/local/bin/python2.7 +#!/usr/local/bin/python3.6 # -*- coding: utf-8 -*- """ - Copyright (c) 2014-2016 Ad Schellevis + Copyright (c) 2014-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/src/opnsense/service/configd_ctl.py b/src/opnsense/service/configd_ctl.py index ae9778804..e99f89b5d 100755 --- a/src/opnsense/service/configd_ctl.py +++ b/src/opnsense/service/configd_ctl.py @@ -1,7 +1,7 @@ -#!/usr/local/bin/python2.7 +#!/usr/local/bin/python3.6 """ - Copyright (c) 2015 Ad Schellevis + Copyright (c) 2015-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -59,10 +59,10 @@ def exec_config_cmd(exec_command): return None try: - sock.send(exec_command) + sock.send(exec_command.encode()) data = [] while True: - line = sock.recv(65536) + line = sock.recv(65536).decode() if line: data.append(line) else: @@ -82,7 +82,7 @@ socket.setdefaulttimeout(120) # validate parameters if len(sys.argv) <= 1: - print ('usage : %s [-m] '%sys.argv[0]) + print('usage : %s [-m] '%sys.argv[0]) sys.exit(0) # check if configd socket exists @@ -95,7 +95,7 @@ while not os.path.exists(configd_socket_name): i += 1 if not os.path.exists(configd_socket_name): - print ('configd socket missing (@%s)'%configd_socket_name) + print('configd socket missing (@%s)'%configd_socket_name) sys.exit(-1) if sys.argv[1] == '-m': diff --git a/src/opnsense/service/modules/addons/template_helpers.py b/src/opnsense/service/modules/addons/template_helpers.py index 8759ae876..587b8a0a0 100644 --- a/src/opnsense/service/modules/addons/template_helpers.py +++ b/src/opnsense/service/modules/addons/template_helpers.py @@ -1,5 +1,5 @@ """ - Copyright (c) 2015 Ad Schellevis + Copyright (c) 2015-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/src/opnsense/service/modules/config.py b/src/opnsense/service/modules/config.py index f357aeed5..ca1529496 100644 --- a/src/opnsense/service/modules/config.py +++ b/src/opnsense/service/modules/config.py @@ -1,5 +1,5 @@ """ - Copyright (c) 2015 Ad Schellevis + Copyright (c) 2015-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/src/opnsense/service/modules/csconfigparser.py b/src/opnsense/service/modules/csconfigparser.py index e893f4f52..d0020530b 100644 --- a/src/opnsense/service/modules/csconfigparser.py +++ b/src/opnsense/service/modules/csconfigparser.py @@ -1,5 +1,5 @@ """ - Copyright (c) 2015 Ad Schellevis + Copyright (c) 2015-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -29,7 +29,7 @@ function: make standard config parser case sensitive """ -from ConfigParser import ConfigParser +from configparser import ConfigParser class CSConfigParser(ConfigParser): diff --git a/src/opnsense/service/modules/ph_inline_actions.py b/src/opnsense/service/modules/ph_inline_actions.py index 57b2b472d..4325305af 100644 --- a/src/opnsense/service/modules/ph_inline_actions.py +++ b/src/opnsense/service/modules/ph_inline_actions.py @@ -1,5 +1,5 @@ """ - Copyright (c) 2014 Ad Schellevis + Copyright (c) 2014-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -30,8 +30,8 @@ """ import syslog -import template -import config +from . import template +from . import config __author__ = 'Ad Schellevis' @@ -86,11 +86,11 @@ def execute(action, parameters): return 'ERR' elif action.command == 'configd.actions': # list all available configd actions - from processhandler import ActionHandler + from .processhandler import ActionHandler act_handler = ActionHandler() actions = act_handler.list_actions(['message', 'description']) - if unicode(parameters).lower() == 'json': + if str(parameters).lower() == 'json': import json return json.dumps(actions) else: diff --git a/src/opnsense/service/modules/processhandler.py b/src/opnsense/service/modules/processhandler.py index 529552833..40500d7f0 100644 --- a/src/opnsense/service/modules/processhandler.py +++ b/src/opnsense/service/modules/processhandler.py @@ -1,5 +1,5 @@ """ - Copyright (c) 2014 Ad Schellevis + Copyright (c) 2014-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -34,13 +34,13 @@ import socket import traceback import syslog import threading -import ConfigParser +import configparser import glob import time import uuid import shlex import tempfile -import ph_inline_actions +from . import ph_inline_actions from modules import singleton __author__ = 'Ad Schellevis' @@ -126,7 +126,7 @@ class Handler(object): return except Exception: # something went wrong... send traceback to syslog, restart listener (wait for a short time) - print (traceback.format_exc()) + print(traceback.format_exc()) syslog.syslog(syslog.LOG_ERR, 'Handler died on %s' % traceback.format_exc()) time.sleep(1) @@ -163,12 +163,12 @@ class HandlerClient(threading.Thread): # noinspection PyBroadException try: # receive command, maximum data length is 4k... longer messages will be truncated - data = self.connection.recv(4096) + data = self.connection.recv(4096).decode() # map command to action data_parts = shlex.split(data) if len(data_parts) == 0 or len(data_parts[0]) == 0: # no data found - self.connection.sendall('no data\n') + self.connection.sendall(('no data\n').encode()) else: exec_command = data_parts[0] if exec_command[0] == "&": @@ -187,7 +187,7 @@ class HandlerClient(threading.Thread): # when running in background, return this message uuid and detach socket if exec_in_background: result = self.message_uuid - self.connection.sendall('%s\n%c%c%c' % (result, chr(0), chr(0), chr(0))) + self.connection.sendall(('%s\n%c%c%c' % (result, chr(0), chr(0), chr(0))).encode()) self.connection.shutdown(socket.SHUT_RDWR) self.connection.close() @@ -200,22 +200,22 @@ class HandlerClient(threading.Thread): if not exec_in_background: # send response back to client( including trailing enter ) - self.connection.sendall('%s\n' % result) + self.connection.sendall(('%s\n' % result).encode()) else: # log response syslog.syslog(syslog.LOG_INFO, "message %s [%s.%s] returned %s " % (self.message_uuid, exec_command, exec_action, - unicode(result)[:100])) + result[:100])) # send end of stream characters if not exec_in_background: - self.connection.sendall("%c%c%c" % (chr(0), chr(0), chr(0))) + self.connection.sendall(("%c%c%c" % (chr(0), chr(0), chr(0))).encode()) except SystemExit: # ignore system exit related errors pass except Exception: - print (traceback.format_exc()) + print(traceback.format_exc()) syslog.syslog( syslog.LOG_ERR, 'unable to sendback response [%s] for [%s][%s][%s] {%s}, message was %s' % (result, @@ -268,7 +268,7 @@ class ActionHandler(object): self.action_map[topic_name] = {} # traverse config directory and open all filenames starting with actions_ - cnf = ConfigParser.RawConfigParser() + cnf = configparser.RawConfigParser() cnf.read(config_filename) for section in cnf.sections(): # map configuration data on object @@ -370,10 +370,10 @@ class ActionHandler(object): :return: None """ action_obj = self.find_action(command, action, parameters) - print ('---------------------------------------------------------------------') - print ('execute %s.%s with parameters : %s ' % (command, action, parameters)) - print ('action object %s (%s) %s' % (action_obj, action_obj.command, message_uuid)) - print ('---------------------------------------------------------------------') + print('---------------------------------------------------------------------') + print('execute %s.%s with parameters : %s ' % (command, action, parameters)) + print('action object %s (%s) %s' % (action_obj, action_obj.command, message_uuid)) + print('---------------------------------------------------------------------') class Action(object): @@ -488,7 +488,7 @@ class Action(object): '[%s] Script action stderr returned "%s"' % (message_uuid, script_error_output.strip()[:255]) ) - return script_output + return script_output.decode() except Exception as script_exception: syslog.syslog(syslog.LOG_ERR, '[%s] Script action failed with %s at %s' % (message_uuid, script_exception, diff --git a/src/opnsense/service/modules/template.py b/src/opnsense/service/modules/template.py index 4a279bb28..a40824c1a 100644 --- a/src/opnsense/service/modules/template.py +++ b/src/opnsense/service/modules/template.py @@ -1,5 +1,5 @@ """ - Copyright (c) 2015 Ad Schellevis + Copyright (c) 2015-2019 Ad Schellevis All rights reserved. Redistribution and use in source and binary forms, with or without @@ -39,7 +39,7 @@ import traceback import copy import codecs import jinja2 -import addons.template_helpers +from .addons import template_helpers __author__ = 'Ad Schellevis' @@ -59,7 +59,6 @@ class Template(object): self._template_dir = os.path.dirname(os.path.abspath(__file__)) + '/../templates/' self._j2_env = jinja2.Environment(loader=jinja2.FileSystemLoader(self._template_dir), trim_blocks=True, extensions=["jinja2.ext.do", "jinja2.ext.loopcontrols"]) - # register additional filters self._j2_env.filters['decode_idna'] = lambda x:x.decode('idna') self._j2_env.filters['encode_idna'] = self._encode_idna @@ -68,7 +67,7 @@ class Template(object): def _encode_idna(x): """ encode string to idna, preserve leading dots """ - return ''.join(map(lambda x:'.', range(len(x) - len(x.lstrip('.'))))) + x.lstrip('.').encode('idna') + return b''.join([b''.join([b'.' for x in range(len(x) - len(x.lstrip('.')))]), x.lstrip('.').encode('idna')]) def list_module(self, module_name): """ list single module content @@ -84,20 +83,21 @@ class Template(object): for target_source in target_sources: if os.path.exists(target_source): - for line in open(target_source, 'r').read().split('\n'): - parts = line.split(':') - if len(parts) > 1 and parts[0].strip()[0] != '#': - source_file = parts[0].strip() - target_name = parts[1].strip() - if target_name in result['+TARGETS'].values(): - syslog.syslog(syslog.LOG_NOTICE, "template overlay %s with %s" % ( - target_name, os.path.basename(target_source) - )) - result['+TARGETS'][source_file] = target_name - if len(parts) == 2: - result['+CLEANUP_TARGETS'][source_file] = target_name - elif parts[2].strip() != "": - result['+CLEANUP_TARGETS'][source_file] = parts[2].strip() + with open(target_source, 'r') as fhandle: + for line in fhandle.read().split('\n'): + parts = line.split(':') + if len(parts) > 1 and parts[0].strip()[0] != '#': + source_file = parts[0].strip() + target_name = parts[1].strip() + if target_name in list(result['+TARGETS'].values()): + syslog.syslog(syslog.LOG_NOTICE, "template overlay %s with %s" % ( + target_name, os.path.basename(target_source) + )) + result['+TARGETS'][source_file] = target_name + if len(parts) == 2: + result['+CLEANUP_TARGETS'][source_file] = target_name + elif parts[2].strip() != "": + result['+CLEANUP_TARGETS'][source_file] = parts[2].strip() return result def list_modules(self): @@ -155,9 +155,9 @@ class Template(object): config_ptr = config_ptr[xmlNodeName] elif xmlNodeName == '%': if type(config_ptr) in (collections.OrderedDict, dict): - target_keys = config_ptr.keys() + target_keys = list(config_ptr) else: - target_keys = map(lambda x: str(x), range(len(config_ptr))) + target_keys = [str(x) for x in range(len(config_ptr))] else: # config pointer is reused when the match is exact, so we need to reset it here # if the tag was not found. @@ -217,15 +217,15 @@ class Template(object): """ result = [] module_data = self.list_module(module_name) - for src_template in module_data['+TARGETS'].keys(): + for src_template in list(module_data['+TARGETS']): target = module_data['+TARGETS'][src_template] target_filename_tags = self.__find_string_tags(target) target_filters = self.__find_filters(target_filename_tags) result_filenames = {target: {}} - for target_filter in target_filters.keys(): - for key in target_filters[target_filter].keys(): - for filename in result_filenames.keys(): + for target_filter in list(target_filters): + for key in list(target_filters[target_filter]): + for filename in list(result_filenames): if target_filters[target_filter][key] is not None \ and filename.find('[%s]' % target_filter) > -1: new_filename = filename.replace('[%s]' % target_filter, target_filters[target_filter][key]) @@ -240,14 +240,14 @@ class Template(object): except jinja2.exceptions.TemplateSyntaxError as templExc: raise Exception("%s %s %s" % (module_name, template_filename, templExc)) - for filename in result_filenames.keys(): + for filename in list(result_filenames): if not (filename.find('[') != -1 and filename.find(']') != -1): # copy config data cnf_data = copy.deepcopy(self._config) cnf_data['TARGET_FILTERS'] = result_filenames[filename] # link template helpers - self._j2_env.globals['helpers'] = addons.template_helpers.Helpers(cnf_data) + self._j2_env.globals['helpers'] = template_helpers.Helpers(cnf_data) # make sure we're only rendering output once if filename not in result: @@ -271,7 +271,7 @@ class Template(object): # It looks like Jinja sometimes isn't consistent on placing this last end-of-line in. if len(content) > 1 and content[-1] != '\n': src_file = '%s%s' % (self._template_dir, template_filename) - src_file_handle = open(src_file, 'r') + src_file_handle = open(src_file, 'rb') src_file_handle.seek(-1, os.SEEK_END) last_bytes_template = src_file_handle.read() src_file_handle.close() @@ -342,7 +342,7 @@ class Template(object): for template_name in self.iter_modules(module_name): syslog.syslog(syslog.LOG_NOTICE, "cleanup template container %s" % template_name) module_data = self.list_module(module_name) - for src_template in module_data['+CLEANUP_TARGETS'].keys(): + for src_template in list(module_data['+CLEANUP_TARGETS']): target = module_data['+CLEANUP_TARGETS'][src_template] for filename in glob.glob(target): os.remove(filename) diff --git a/src/opnsense/service/run_unittests.py b/src/opnsense/service/run_unittests.py index 5935c025b..fef5e0f6d 100755 --- a/src/opnsense/service/run_unittests.py +++ b/src/opnsense/service/run_unittests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3.6 """ Copyright (c) 2016 Ad Schellevis All rights reserved. diff --git a/src/opnsense/service/tests/config/config.xml b/src/opnsense/service/tests/config/config.xml index 643af571d..bf5046c3a 100644 --- a/src/opnsense/service/tests/config/config.xml +++ b/src/opnsense/service/tests/config/config.xml @@ -1,5 +1,6 @@ + 1 diff --git a/src/opnsense/service/tests/core.py b/src/opnsense/service/tests/core.py index 52f0a1953..d55d2dcb6 100644 --- a/src/opnsense/service/tests/core.py +++ b/src/opnsense/service/tests/core.py @@ -57,14 +57,14 @@ class DummySocket(object): :param size: :return: """ - return self._send_data + return self._send_data.encode() def sendall(self, data): """ send back to "client" :param data: text :return: """ - self._receive_data.append(data) + self._receive_data.append(data.decode()) def close(self): """ close connection @@ -78,6 +78,9 @@ class DummySocket(object): """ return ''.join(self._receive_data) + def shutdown(self, mode): + pass + class TestCoreMethods(unittest.TestCase): def setUp(self): @@ -106,7 +109,7 @@ class TestCoreMethods(unittest.TestCase): action_handler=self.act_handler, simulation_mode=False) cmd_thread.run() - self.assertEquals(self.dummysock.getReceived()[-4:], '\n%c%c%c' % (chr(0), chr(0), chr(0)), "Invalid sequence") + self.assertEqual(self.dummysock.getReceived()[-4:], '\n%c%c%c' % (chr(0), chr(0), chr(0)), "Invalid sequence") def test_command_unknown(self): """ test invalid command @@ -118,7 +121,7 @@ class TestCoreMethods(unittest.TestCase): action_handler=self.act_handler, simulation_mode=False) cmd_thread.run() - self.assertEquals(self.dummysock.getReceived().split('\n')[0], 'Action not found', 'Invalid response') + self.assertEqual(self.dummysock.getReceived().split('\n')[0], 'Action not found', 'Invalid response') def test_configd_actions(self): """ request configd command list diff --git a/src/opnsense/service/tests/template.py b/src/opnsense/service/tests/template.py index 86a41f735..93f696edd 100644 --- a/src/opnsense/service/tests/template.py +++ b/src/opnsense/service/tests/template.py @@ -52,7 +52,7 @@ class TestConfigMethods(unittest.TestCase): """ test correct config type :return: """ - self.assertEquals(type(self.conf.get()), collections.OrderedDict) + self.assertEqual(type(self.conf.get()), collections.OrderedDict) def test_interface(self): """ test existence of interface @@ -93,8 +93,8 @@ class TestTemplateMethods(unittest.TestCase): """ test sample template :return: """ - generated_filenames = self.tmpl.generate('OPNsense.Sample') - self.assertEquals(len(generated_filenames), 3, 'number of output files not 3') + generated_filenames = self.tmpl.generate('OPNsense/Sample') + self.assertEqual(len(generated_filenames), 4, 'number of output files not 4') def test_all(self): """ Test if all expected templates are created, can only find test for static defined cases. @@ -109,11 +109,14 @@ class TestTemplateMethods(unittest.TestCase): for filenm in files: if filenm == '+TARGETS': filename = '%s/%s' % (root, filenm) - for line in open(filename).read().split('\n'): - line = line.strip() - if len(line) > 1 and line[0] != '#' and line.find('[') == -1: - expected_filename = ('%s%s' % (self.output_path, line.split(':')[-1])).replace('//', '/') - self.expected_filenames[expected_filename] = {'src': filename} + with open(filename) as fhandle: + for line in fhandle.read().split('\n'): + line = line.strip() + if len(line) > 1 and line[0] != '#' and line.find('[') == -1: + expected_filename = ( + '%s%s' % (self.output_path, line.split(':')[-1]) + ).replace('//', '/') + self.expected_filenames[expected_filename] = {'src': filename} for filename in self.tmpl.generate('*'): self.generated_filenames.append(filename.replace('//', '/'))