Proxy: custom error pages (https://github.com/opnsense/core/issues/4174)

o add template download configd call
o align controller and ui to use the download call (flush config to disk, request "active" error_pages)
o refactor deploy_error_pages.py to ease download_error_pages.py implementation

src/opnsense/mvc/app/controllers/OPNsense/Proxy/Api/TemplateController.php

@@ -29,6 +29,7 @@
 namespace OPNsense\Proxy\Api;
 
 use OPNsense\Base\ApiMutableModelControllerBase;
+use OPNsense\Core\Backend;
 
 /**
  * Class TemplateController
@@ -81,13 +82,21 @@ class TemplateController extends ApiMutableModelControllerBase
     }
 
     /**
-     * retrieve error pages template
+     * retrieve error pages template, overlay provided template zip file on top of OPNsense error pages
+     * using configd calls
      */
     public function getAction()
     {
-        $mdl = $this->getModel();
-        return [
-            'content' => (string)$mdl->error_pages->template
-        ];
+        $backend = new Backend();
+        $backend->configdRun("template reload OPNsense/Proxy");
+        $result = json_decode($backend->configdRun("proxy download_error_pages"), true);
+        if ($result != null) {
+            $this->response->setRawHeader("Content-Type: application/octet-stream");
+            $this->response->setRawHeader("Content-Disposition: attachment; filename=proxy_template.zip");
+            return base64_decode($result['payload']);
+        } else {
+            // return empty response on error
+            return "";
+        }
     }
 }

src/opnsense/mvc/app/views/OPNsense/Proxy/index.volt

@@ -282,21 +282,7 @@
              }
          });
          $("#error_pages_download").click(function(){
-             ajaxGet("/api/proxy/template/get", {}, function(data, status){
-                 if (data.content) {
-                   let a_tag = $('<a></a>').attr('href','data:application/zip;charset=utf8,' + encodeURIComponent(data.content))
-                       .attr('download','proxy_template.zip').appendTo('body');
-
-                   a_tag.ready(function() {
-                       if ( window.navigator.msSaveOrOpenBlob && window.Blob ) {
-                           var blob = new Blob( [ output_data ], { type: "application/zip" } );
-                           navigator.msSaveOrOpenBlob( blob, 'proxy_template.zip' );
-                       } else {
-                           a_tag.get(0).click();
-                       }
-                   });
-                 }
-             });
+            window.open('/api/proxy/template/get', 'downloadTemplate');
          });
          $("#error_pages_upload").click(function(){
              if ($("#error_pages_content").val().length > 2) {

src/opnsense/scripts/proxy/deploy_error_pages.py

@@ -39,17 +39,16 @@ if __name__ == '__main__':
     if not os.path.isdir(target_directory):
         os.mkdir(target_directory)
     for filename, data in proxy_templates.templates(proxy_templates.overlay_enabled()):
-        if filename.endswith('.html'):
-            match = re.search(b'(<!--[\s]*EMBED:start.*?EMBED:end[\s]*-->)', data, re.DOTALL)
-            if match:
-                inline_css = list()
-                for href in re.findall(b"(href[\s]*=[\s]*[\"|'])(.*?)([\"|'])" ,match.group(0)):
-                    href_content = proxy_templates.get_file(href[1].decode(), proxy_templates.overlay_enabled())
-                    if href_content:
-                        inline_css.append(b'<style type="text/css">\n%s\n</style>' % href_content)
-                data = b"%s%s%s" % (data[0:match.start()], b"\n".join(inline_css), data[match.end():])
-                with open("%s/%s" % (target_directory, os.path.splitext(filename)[0]), "wb") as target_fh:
-                    target_fh.write(data)
+        match = proxy_templates.css_section(data)
+        if match:
+            inline_css = list()
+            for dep_filename in proxy_templates.css_dependencies(filename, proxy_templates.overlay_enabled()):
+                css_content = proxy_templates.get_file(dep_filename, proxy_templates.overlay_enabled())
+                if css_content:
+                    inline_css.append(b'<style type="text/css">\n%s\n</style>' % css_content)
+            data = b"%s%s%s" % (data[0:match.start()], b"\n".join(inline_css), data[match.end():])
+        with open("%s/%s" % (target_directory, os.path.splitext(filename)[0]), "wb") as target_fh:
+            target_fh.write(data)
     print(ujson.dumps({
         'overlay_status': proxy_templates.get_overlay_status()
     }))

src/opnsense/scripts/proxy/download_error_pages.py

@@ -0,0 +1,53 @@
+#!/usr/local/bin/python3
+
+"""
+    Copyright (c) 2020 Ad Schellevis <ad@opnsense.org>
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+
+"""
+import base64
+import ujson
+import os
+import re
+import zipfile
+from io import BytesIO
+from lib import ProxyTemplates
+
+if __name__ == '__main__':
+    root_dir = "/proxy_template"
+    proxy_templates = ProxyTemplates()
+    output_data = BytesIO()
+    processed = list()
+    with zipfile.ZipFile(output_data, mode='w', compression=zipfile.ZIP_DEFLATED) as zf:
+        for filename, data in proxy_templates.templates(True):
+            zf.writestr("%s/%s" % (root_dir, filename), data)
+            for dep_filename in proxy_templates.css_dependencies(filename, True):
+                if dep_filename not in processed:
+                    zf.writestr("%s/%s" % (root_dir, dep_filename), proxy_templates.get_file(dep_filename, True))
+                    processed.append(dep_filename)
+
+    response = dict()
+    response['payload'] = base64.b64encode(output_data.getvalue()).decode()
+    response['size'] = len(response['payload'])
+    print(ujson.dumps(response))

src/opnsense/scripts/proxy/lib/__init__.py

@@ -28,6 +28,7 @@ import ujson
 import os
 import base64
 import binascii
+import re
 import zipfile
 import glob
 from io import BytesIO
@@ -45,12 +46,16 @@ class ProxyTemplates:
         self.load()
 
     def _load_config(self):
+        """ initialize configuration
+        """
         if os.path.isfile(self.error_config):
             error_cfg = ujson.loads(open(self.error_config, 'rb').read())
             self._install_overlay = 'install' not in error_cfg or error_cfg['install'] != 'opnsense'
             self._overlay_data = error_cfg['content'] if 'content' in error_cfg else None
 
     def load(self):
+        """ load (custom) error pages in memory
+        """
         self._overlay_status = None
         self._all_src_files = dict()
         self._all_ovl_files = dict()
@@ -79,21 +84,58 @@ class ProxyTemplates:
                 self._overlay_status = 'Error reading file'
 
     def templates(self, overlay=False):
+        """ return template html files
+            :param overlay: consider custom theme files when applicable
+            :rtype: [string, bytes]
+        """
         for filename in self._all_src_files:
-            if overlay and filename in self._all_ovl_files:
-                yield filename, self._all_ovl_files[filename]
-            else:
-                yield filename, self._all_src_files[filename]
+            if filename.endswith('.html'):
+                if overlay and filename in self._all_ovl_files:
+                    yield filename, self._all_ovl_files[filename]
+                else:
+                    yield filename, self._all_src_files[filename]
 
     def get_file(self, filename, overlay=False):
+        """ return file content
+            :param filename: source filename
+            :param overlay: consider custom theme files when applicable
+            :return: string
+        """
         if filename in self._all_src_files:
             if overlay and filename in self._all_ovl_files:
                 return self._all_ovl_files[filename]
             else:
                 return self._all_src_files[filename]
 
+    @staticmethod
+    def css_section(data):
+        """ extract css definition block from provided data
+            :param data: html data
+            :return: MatchObject
+        """
+        return re.search(b'(<!--[\s]*EMBED:start.*?EMBED:end[\s]*-->)', data, re.DOTALL)
+
+    def css_dependencies(self, filename, overlay=False):
+        """ extract css dependencies from provided filename
+            :param filename: source filename
+            :param overlay: consider custom theme files when applicable
+            :rtype: list
+        """
+        data = self.get_file(filename, overlay)
+        if filename.endswith('.html') and data:
+            match = self.css_section(data)
+            if match:
+                for href in re.findall(b"(href[\s]*=[\s]*[\"|'])(.*?)([\"|'])" ,match.group(0)):
+                    yield href[1].decode()
+
     def overlay_enabled(self):
+        """ when deploying files, should we consider an overlay
+            :return: bool
+        """
         return self._install_overlay
 
     def get_overlay_status(self):
+        """ return validity of the installed overlay
+            :return: string
+        """
         return self._overlay_status

src/opnsense/service/conf/actions.d/actions_proxy.conf

@@ -76,3 +76,10 @@ command:/usr/local/opnsense/scripts/proxy/deploy_error_pages.py
 parameters:
 type:script_output
 message:deploy error pages
+
+
+[download_error_pages]
+command:/usr/local/opnsense/scripts/proxy/download_error_pages.py
+parameters:
+type:script_output
+message:download error pages