diff --git a/ursh/indico_ursh/controllers.py b/ursh/indico_ursh/controllers.py
index 44514b4..6c5c6ac 100644
--- a/ursh/indico_ursh/controllers.py
+++ b/ursh/indico_ursh/controllers.py
@@ -11,7 +11,7 @@ import posixpath
 
 from flask import jsonify, request, session
 from flask_pluginengine import render_plugin_template
-from werkzeug.exceptions import BadRequest
+from werkzeug.exceptions import BadRequest, NotFound
 from werkzeug.urls import url_parse
 
 from indico.core.config import config
@@ -20,7 +20,7 @@ from indico.web.rh import RH
 from indico.web.util import jsonify_template
 
 from indico_ursh import _
-from indico_ursh.util import register_shortcut, request_short_url, strip_end
+from indico_ursh.util import is_configured, register_shortcut, request_short_url, strip_end
 from indico_ursh.views import WPShortenURLPage
 
 
@@ -51,9 +51,11 @@ class RHGetShortURL(RH):
 
 
 class RHShortURLPage(RH):
-    """Provide a simple page, where users can submit a URL to be shortened"""
+    """Provide a simple page where users can submit a URL to be shortened"""
 
     def _process(self):
+        if not is_configured():
+            raise NotFound('Plugin is not configured')
         return WPShortenURLPage.render_template('ursh_shortener_page.html')
 
 
diff --git a/ursh/indico_ursh/plugin.py b/ursh/indico_ursh/plugin.py
index 37da0b8..9ef9f5c 100644
--- a/ursh/indico_ursh/plugin.py
+++ b/ursh/indico_ursh/plugin.py
@@ -18,6 +18,7 @@ from indico.web.views import WPBase
 
 from indico_ursh import _
 from indico_ursh.blueprint import blueprint
+from indico_ursh.util import is_configured
 
 
 class SettingsForm(IndicoForm):
@@ -50,9 +51,10 @@ class UrshPlugin(IndicoPlugin):
         return blueprint
 
     def _inject_ursh_link(self, target=None, event=None, dropdown=False, element_class='', text='', **kwargs):
-        if self.settings.get('api_key') and self.settings.get('api_host') and (target or event):
+        if is_configured() and (target or event):
             return render_plugin_template('ursh_link.html', target=target, event=event,
                                           dropdown=dropdown, element_class=element_class, text=text, **kwargs)
 
     def _inject_ursh_footer(self, **kwargs):
-        return render_plugin_template('ursh_footer.html')
+        if is_configured():
+            return render_plugin_template('ursh_footer.html')
diff --git a/ursh/indico_ursh/util.py b/ursh/indico_ursh/util.py
index 199adc2..b58b0c5 100644
--- a/ursh/indico_ursh/util.py
+++ b/ursh/indico_ursh/util.py
@@ -25,6 +25,14 @@ def _get_settings():
     return api_key, api_host
 
 
+def is_configured():
+    """Check whether the plugin is properly configured."""
+    from indico_ursh.plugin import UrshPlugin
+    api_key = UrshPlugin.settings.get('api_key')
+    api_host = UrshPlugin.settings.get('api_host')
+    return bool(api_key and api_host)
+
+
 def request_short_url(original_url):
     from indico_ursh.plugin import UrshPlugin
     api_key, api_host = _get_settings()