Storage/S3: Add API to get list of buckets

storage_s3/indico_storage_s3/__init__.py

@@ -17,6 +17,10 @@
 from __future__ import unicode_literals
 
 from indico.core import signals
+from indico.util.i18n import make_bound_gettext
+
+
+_ = make_bound_gettext('search_cern')
 
 
 @signals.import_tasks.connect

storage_s3/indico_storage_s3/blueprint.py

@@ -0,0 +1,25 @@
+# This file is part of Indico.
+# Copyright (C) 2002 - 2018 European Organization for Nuclear Research (CERN).
+#
+# Indico is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 3 of the
+# License, or (at your option) any later version.
+#
+# Indico is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Indico; if not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import unicode_literals
+
+from indico.core.plugins import IndicoPluginBlueprint
+
+from indico_storage_s3.controllers import RHBuckets
+
+
+blueprint = IndicoPluginBlueprint('storage_s3', __name__, url_prefix='/api/plugin/s3')
+blueprint.add_url_rule('/buckets', 'buckets', RHBuckets)

storage_s3/indico_storage_s3/controllers.py

@@ -0,0 +1,77 @@
+# This file is part of Indico.
+# Copyright (C) 2002 - 2018 European Organization for Nuclear Research (CERN).
+#
+# Indico is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 3 of the
+# License, or (at your option) any later version.
+#
+# Indico is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Indico; if not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import unicode_literals
+
+from flask import current_app, jsonify, request
+from werkzeug.exceptions import NotFound, Unauthorized
+
+from indico.core.config import config
+from indico.core.db import db
+from indico.core.storage import StoredFileMixin
+from indico.core.storage.backend import ReadOnlyStorageMixin, get_storage
+from indico.web.rh import RH
+
+from indico_storage_s3.storage import DynamicS3Storage, S3Storage, S3StorageBase
+
+
+class RHBuckets(RH):
+    """Provide information on used S3 buckets"""
+
+    def _check_access(self):
+        from indico_storage_s3.plugin import S3StoragePlugin
+        auth = request.authorization
+        if not S3StoragePlugin.settings.get('bucket_info_enabled'):
+            raise NotFound
+        username = S3StoragePlugin.settings.get('username')
+        password = S3StoragePlugin.settings.get('password')
+        if not auth or not auth.password or auth.username != username or auth.password != password:
+            response = current_app.response_class('Authorization required', 401,
+                                                  {'WWW-Authenticate': 'Basic realm="Indico - S3 Buckets"'})
+            raise Unauthorized(response=response)
+
+    def _get_static_info(self, storage):
+        return {
+            'bucket': storage.bucket_name,
+            'dynamic': False,
+        }
+
+    def _get_dynamic_info(self, backend_name, storage):
+        buckets = set()
+        for model in StoredFileMixin.__subclasses__():
+            query = (db.session.query(db.func.split_part(model.storage_file_id, '//', 1))
+                     .filter(model.storage_file_id.isnot(None), model.storage_backend == backend_name))
+            buckets.update(bucket for bucket, in query)
+
+        return {
+            'buckets': sorted(buckets),
+            'dynamic': True,
+            'template': storage.bucket_name_template,
+        }
+
+    def _process(self):
+        data = {}
+        for key in config.STORAGE_BACKENDS:
+            storage = get_storage(key)
+            if not isinstance(storage, S3StorageBase):
+                continue
+            readonly = isinstance(storage, ReadOnlyStorageMixin)
+            data[key] = {'readonly': readonly}
+            if isinstance(storage, S3Storage):
+                data[key].update(self._get_static_info(storage))
+            elif isinstance(storage, DynamicS3Storage):
+                data[key].update(self._get_dynamic_info(key, storage))
+        return jsonify(data)

storage_s3/indico_storage_s3/plugin.py

@@ -19,23 +19,56 @@ from __future__ import unicode_literals
 import sys
 
 import click
+from markupsafe import Markup
+from wtforms.fields import BooleanField, StringField
+from wtforms.validators import DataRequired
 
 from indico.cli.core import cli_command
 from indico.core import signals
 from indico.core.config import config
-from indico.core.plugins import IndicoPlugin
+from indico.core.plugins import IndicoPlugin, url_for_plugin
 from indico.core.storage.backend import ReadOnlyStorageMixin, get_storage
+from indico.web.forms.base import IndicoForm
+from indico.web.forms.fields import IndicoPasswordField
+from indico.web.forms.validators import HiddenUnless
+from indico.web.forms.widgets import SwitchWidget
 
+from indico_storage_s3 import _
+from indico_storage_s3.blueprint import blueprint
 from indico_storage_s3.storage import (DynamicS3Storage, ReadOnlyDynamicS3Storage, ReadOnlyS3Storage, S3Storage,
                                        S3StorageBase)
 
 
+class SettingsForm(IndicoForm):
+    bucket_info_enabled = BooleanField(_("Bucket info API"), widget=SwitchWidget())
+    username = StringField(_("Username"), [HiddenUnless('bucket_info_enabled', preserve_data=True), DataRequired()],
+                           description=_("The username to access the S3 bucket info endpoint"))
+    password = IndicoPasswordField(_('Password'),
+                                   [HiddenUnless('bucket_info_enabled', preserve_data=True), DataRequired()],
+                                   toggle=True,
+                                   description=_("The password to access the S3 bucket info endpoint"))
+
+    def __init__(self, *args, **kwargs):
+        super(SettingsForm, self).__init__(*args, **kwargs)
+        url = Markup('<strong><code>{}</code></strong>').format(url_for_plugin('storage_s3.buckets'))
+        self.bucket_info_enabled.description = _("Enables an API on {url} that returns information on all S3 buckets "
+                                                 "currently in use, including dynamically-named ones.").format(url=url)
+
+
 class S3StoragePlugin(IndicoPlugin):
     """S3 Storage
 
     Provides S3 storage backends.
     """
 
+    configurable = True
+    settings_form = SettingsForm
+    default_settings = {
+        'bucket_info_enabled': False,
+        'username': '',
+        'password': ''
+    }
+
     def init(self):
         super(S3StoragePlugin, self).init()
         self.connect(signals.get_storage_backends, self._get_storage_backends)
@@ -47,6 +80,9 @@ class S3StoragePlugin(IndicoPlugin):
         yield ReadOnlyS3Storage
         yield ReadOnlyDynamicS3Storage
 
+    def get_blueprints(self):
+        return blueprint
+
     def _extend_indico_cli(self, sender, **kwargs):
         @cli_command()
         @click.option('--storage', default=None, metavar='NAME', help='Storage backend to create bucket for')