Export organization data #450

babel.cfg

@@ -1,4 +1,3 @@
 [ignore: env/**]
 [python: project/**.py]
 [jinja2: project/templates/**.html]
-extensions=jinja2.ext.autoescape,jinja2.ext.with_

project/__init__.py

@@ -132,6 +132,7 @@ cache_path = (
     cache_env if os.path.isabs(cache_env) else os.path.join(app.root_path, cache_env)
 )
 dump_path = os.path.join(cache_path, "dump")
+dump_org_path = os.path.join(cache_path, "dump_org")
 img_path = os.path.join(cache_path, "img")
 sitemap_file = "sitemap.xml"
 robots_txt_file = "robots.txt"

project/celery_tasks.py

@@ -6,6 +6,7 @@ from project import celery
 @celery.on_after_configure.connect
 def setup_periodic_tasks(sender, **kwargs):
     sender.add_periodic_task(crontab(hour=0, minute=0), clear_images_task)
+    sender.add_periodic_task(crontab(hour=0, minute=5), clear_admin_unit_dumps_task)
     sender.add_periodic_task(crontab(hour=1, minute=0), update_recurring_dates_task)
     sender.add_periodic_task(crontab(hour=2, minute=0), dump_all_task)
     sender.add_periodic_task(crontab(hour=3, minute=0), seo_generate_sitemap_task)
@@ -52,6 +53,16 @@ def dump_admin_unit_task(admin_unit_id):
     dump_admin_unit(admin_unit_id)
 
 
+@celery.task(
+    acks_late=True,
+    reject_on_worker_lost=True,
+)
+def clear_admin_unit_dumps_task():
+    from project.services.dump import clear_admin_unit_dumps
+
+    clear_admin_unit_dumps()
+
+
 @celery.task(
     acks_late=True,
     reject_on_worker_lost=True,

project/cli/dump.py

@@ -1,3 +1,4 @@
+import click
 from flask.cli import AppGroup
 
 from project import app
@@ -11,4 +12,10 @@ def dump_all():
     dump.dump_all()
 
 
+@dump_cli.command("organization")
+@click.argument("admin_unit_id")
+def dump_admin_unit(admin_unit_id):
+    dump.dump_admin_unit(admin_unit_id)
+
+
 app.cli.add_command(dump_cli)

project/jinja_filters.py

@@ -35,6 +35,16 @@ def ensure_link_scheme(link: str):
     return f"https://{link}"
 
 
+def human_file_size(bytes, units=[" bytes", "KB", "MB", "GB", "TB", "PB", "EB"]):
+    return (
+        str(bytes) + units[0]
+        if bytes < 1024
+        else human_file_size(bytes >> 10, units[1:])
+        if units[1:]
+        else f"{bytes>>10}ZB"
+    )
+
+
 app.jinja_env.filters["event_category_name"] = lambda u: get_event_category_name(u)
 app.jinja_env.filters["loc_enum"] = lambda u: get_localized_enum_name(u)
 app.jinja_env.filters["loc_scope"] = lambda s: get_localized_scope(s)
@@ -45,6 +55,7 @@ app.jinja_env.filters["any_dict_value_true"] = any_dict_value_true
 app.jinja_env.filters["ensure_link_scheme"] = lambda s: ensure_link_scheme(s)
 app.jinja_env.filters["place_str"] = lambda p: get_place_str(p)
 app.jinja_env.filters["location_str"] = lambda location: get_location_str(location)
+app.jinja_env.filters["human_file_size"] = lambda size: human_file_size(size)
 
 
 def get_base_url():

project/services/dump.py

@@ -5,7 +5,7 @@ import shutil
 from sqlalchemy import and_
 from sqlalchemy.orm import joinedload
 
-from project import app, db, dump_path
+from project import app, db, dump_org_path, dump_path
 from project.api.event.schemas import EventDumpSchema
 from project.api.event_category.schemas import EventCategoryDumpSchema
 from project.api.event_reference.schemas import EventReferenceDumpSchema
@@ -22,12 +22,12 @@ from project.models import (
     EventReference,
     PublicStatus,
 )
-from project.utils import make_dir
+from project.utils import clear_files_in_dir, make_dir
 
 
 class Dumper(object):
-    def __init__(self, dump_path, file_base_name):
-        self.dump_path = dump_path
+    def __init__(self, dump_base_path, file_base_name):
+        self.dump_base_path = dump_base_path
         self.file_base_name = file_base_name
         self.tmp_path = None
 
@@ -91,7 +91,7 @@ class Dumper(object):
 
         app.logger.info(f"{len(items)} item(s) dumped to {path}.")
 
-    def dump_item(self, items, schema, file_base_name):  # pragma: no cover
+    def dump_item(self, items, schema, file_base_name):
         result = schema.dump(items)
         path = os.path.join(self.tmp_path, file_base_name + ".json")
 
@@ -101,18 +101,18 @@ class Dumper(object):
         app.logger.info(f"Item dumped to {path}.")
 
     def setup_tmp_dir(self):
-        self.tmp_path = os.path.join(self.dump_path, f"tmp-{self.file_base_name}")
+        self.tmp_path = os.path.join(self.dump_base_path, f"tmp-{self.file_base_name}")
         make_dir(self.tmp_path)
 
     def clean_up_tmp_dir(self):
         shutil.rmtree(self.tmp_path, ignore_errors=True)
 
     def zip_tmp_dir(self):
-        zip_base_name = os.path.join(dump_path, self.file_base_name)
+        zip_base_name = os.path.join(self.dump_base_path, self.file_base_name)
         zip_path = shutil.make_archive(zip_base_name, "zip", self.tmp_path)
         app.logger.info(f"Zipped all up to {zip_path}.")
 
-    def dump_image(self, image):  # pragma: no cover
+    def dump_image(self, image):
         if not image:
             return
 
@@ -121,9 +121,9 @@ class Dumper(object):
         get_image_from_bytes(image.data).save(file_path)
 
 
-class AdminUnitDumper(Dumper):  # pragma: no cover
-    def __init__(self, dump_path, admin_unit_id):
-        super().__init__(dump_path, f"org-{admin_unit_id}")
+class AdminUnitDumper(Dumper):
+    def __init__(self, dump_base_path, admin_unit_id):
+        super().__init__(dump_base_path, f"org-{admin_unit_id}")
         self.admin_unit_id = admin_unit_id
 
     def dump_data(self):
@@ -173,6 +173,12 @@ def dump_all():
     dumper.dump()
 
 
-def dump_admin_unit(admin_unit_id):  # pragma: no cover
-    dumper = AdminUnitDumper(dump_path, admin_unit_id)
+def dump_admin_unit(admin_unit_id):
+    dumper = AdminUnitDumper(dump_org_path, admin_unit_id)
     dumper.dump()
+
+
+def clear_admin_unit_dumps():
+    app.logger.info("Clearing admin unit dumps..")
+    clear_files_in_dir(dump_org_path)
+    app.logger.info("Done.")

project/templates/_macros.html

@@ -1687,13 +1687,13 @@ $('#allday').on('change', function() {
 </script>
 {% endmacro %}
 
-{% macro render_form_scripts() %}
-<script src="{{ url_for('static', filename='ext/jquery-ui.1.12.1/jquery-ui.min.js')}}"></script>
-<script src="{{ url_for('static', filename='ext/jquery-ui-i18n.1.11.4.min.js')}}"></script>
-<script src="{{ url_for('static', filename='ext/select2.4.1.0-beta.1.min.js')}}"></script>
-<script src="{{ url_for('static', filename='ext/select2.i18n.de.4.1.0-beta.1.min.js')}}"></script>
-<script src="{{ url_for('static', filename='ext/jquery.timepicker.1.13.18.min.js')}}"></script>
+{% macro render_ajax_csrf_script() %}
 <script type="text/javascript">
+  {{ render_ajax_csrf() }}
+</script>
+{% endmacro %}
+
+{% macro render_ajax_csrf() %}
   var csrf_token = "{{ csrf_token() }}";
 
   $.ajaxSetup({
@@ -1703,6 +1703,16 @@ $('#allday').on('change', function() {
           }
       }
   });
+{% endmacro %}
+
+{% macro render_form_scripts() %}
+<script src="{{ url_for('static', filename='ext/jquery-ui.1.12.1/jquery-ui.min.js')}}"></script>
+<script src="{{ url_for('static', filename='ext/jquery-ui-i18n.1.11.4.min.js')}}"></script>
+<script src="{{ url_for('static', filename='ext/select2.4.1.0-beta.1.min.js')}}"></script>
+<script src="{{ url_for('static', filename='ext/select2.i18n.de.4.1.0-beta.1.min.js')}}"></script>
+<script src="{{ url_for('static', filename='ext/jquery.timepicker.1.13.18.min.js')}}"></script>
+<script type="text/javascript">
+  {{ render_ajax_csrf() }}
 
   $.datepicker.setDefaults($.datepicker.regional["de"]);
   $.fn.select2.defaults.set("language", "de");

project/templates/developer/read.html

@@ -5,7 +5,7 @@
 {%- endblock -%}
 {% block content %}
 
-<h1>Developer</h1>
+<h1>{{ _('Developer') }}</h1>
 
 <h2>API</h2>
 <ul>
@@ -13,13 +13,13 @@
     <li>Code generation: <a href="https://editor.swagger.io/?url={{ url_for('home', _external=True) }}swagger/" target="_blank" rel="noopener noreferrer">Swagger Editor</a></li>
 </ul>
 
-<h2>Data download</h2>
+<h2>{{ _('Download') }}</h2>
 <ul>
     <li>
     {% if dump_file %}
-        <a href="{{ dump_file.url }}">Dump of all data</a> <span class="badge badge-pill badge-light">{{ dump_file.ctime | datetimeformat }}</span> <span class="badge badge-pill badge-light">{{ dump_file.size }} Bytes</span>
+        <a href="{{ dump_file.url }}">{{ _('All data') }}</a> <span class="badge badge-pill badge-light">{{ dump_file.ctime | datetimeformat }}</span> <span class="badge badge-pill badge-light">{{ dump_file.size | human_file_size }}</span>
     {% else %}
-        No files available
+        {{ _('No files available') }}
     {% endif %}
     </li>
     <li>The data file format is part of the <a href="/swagger-ui" target="_blank" rel="noopener noreferrer">API spec</a>. Watch for the <code>*Dump</code> models.</li>

project/templates/layout.html

@@ -255,9 +255,13 @@
                     {% if current_admin_unit.can_invite_other %}
                     <a class="dropdown-item" href="{{ url_for('manage_admin_unit_organization_invitations', id=current_admin_unit.id) }}">{{ _('Organization invitations') }}</a>
                     {% endif %}
+                    <div class="dropdown-divider"></div>
                     <a class="dropdown-item" href="{{ url_for('admin_unit_update', id=current_admin_unit.id) }}">{{ _('Settings') }}</a>
+                    <a class="dropdown-item" href="{{ url_for('manage_admin_unit_export', id=current_admin_unit.id) }}">{{ _('Export') }}</a>
+                    <div class="dropdown-divider"></div>
                     <a class="dropdown-item" href="{{ url_for('manage_admin_unit_custom_widgets', id=current_admin_unit.id) }}">{{ _('Custom widgets') }}</a>
                     <a class="dropdown-item" href="{{ url_for('manage_admin_unit_widgets', id=current_admin_unit.id) }}">{{ _('Widgets') }}</a>
+                    <div class="dropdown-divider"></div>
                     <a class="dropdown-item" href="{{ url_for('organizations', path=current_admin_unit.id) }}">{{ _('Profile') }}</a>
                   </div>
                 </li>

project/templates/manage/export.html

@@ -0,0 +1,90 @@
+{% extends "layout.html" %}
+{% from "_macros.html" import render_ajax_csrf_script %}
+{%- block title -%}
+{{ _('Export') }}
+{%- endblock -%}
+{% block header %}
+{{ render_ajax_csrf_script() }}
+<script>
+    function submit_async() {
+        $("#submit_async"). prop("disabled", true);
+        handle_request_start();
+        $.ajax({
+            url: "{{ url_for('manage_admin_unit_export', id=admin_unit.id) }}",
+            type: "post",
+            dataType: "json",
+            error: function(xhr, status, error) {
+                $("#submit_async"). prop("disabled", false);
+                handle_request_error(xhr, status, error);
+            },
+            success: function (data) {
+                poll(data["result_id"]);
+            }
+        });
+    }
+
+    function poll(result_id) {
+        $.ajax({
+            url: "{{ url_for('manage_admin_unit_export', id=admin_unit.id) }}",
+            type: "get",
+            dataType: "json",
+            data: "poll=" + result_id,
+            error: function(xhr, status, error) {
+                $("#submit_async"). prop("disabled", false);
+                handle_request_error(xhr, status, error);
+            },
+            success: function (data) {
+                if (!data["ready"]) {
+                    setTimeout(function() {
+                        poll(result_id);
+                    }, 2000);
+                    return;
+                }
+
+                if (!data["successful"]) {
+                  console.error(data);
+                  handle_request_error(null, JSON.stringify(data), data);
+                  return;
+                }
+
+                window.location.reload();
+            }
+        });
+    }
+
+    $( function() {
+        $("#submit_async").click(function(){
+            submit_async();
+            return false;
+        });
+    });
+</script>
+{% endblock %}
+{% block content %}
+
+<h1>{{ _('Export') }}</h1>
+
+<h2>{{ _('Download') }}</h2>
+<ul>
+    <li>
+    {% if dump_file %}
+        <a href="{{ dump_file.url }}">{{ _('All data') }}</a> <span class="badge badge-pill badge-light">{{ dump_file.ctime | datetimeformat }}</span> <span class="badge badge-pill badge-light">{{ dump_file.size | human_file_size }}</span>
+    {% else %}
+        {{ _('No files available') }}
+    {% endif %}
+    </li>
+</ul>
+
+<p>
+    <button id="submit_async" type="button" class="btn btn-primary">{{ _('Create export files') }}</button>
+    <div class="col-md">
+        <div id="result_container">
+        </div>
+        <div class="spinner-border m-3" role="status" id="spinner" style="display: none;">
+            <span class="sr-only">Loading&hellip;</span>
+        </div>
+        <div class="alert alert-danger m-3" role="alert" id="error_alert" style="display: none;"></div>
+    </div>
+</p>
+
+{% endblock %}

project/utils.py

@@ -56,6 +56,9 @@ def make_dir(path):
 
 
 def clear_files_in_dir(path):
+    if not os.path.exists(path):  # pragma: no cover
+        return
+
     with os.scandir(path) as entries:
         for entry in entries:
             if entry.is_file() or entry.is_symlink():

project/views/admin.py

@@ -5,7 +5,7 @@ from flask_security import roles_required
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.sql import func
 
-from project import app, celery, db, user_datastore
+from project import app, db, user_datastore
 from project.base_tasks import send_mail_task
 from project.forms.admin import (
     AdminNewsletterForm,
@@ -21,6 +21,8 @@ from project.services.admin import upsert_settings
 from project.services.user import set_roles_for_user
 from project.views.utils import (
     flash_errors,
+    get_celery_poll_group_result,
+    get_celery_poll_result,
     get_pagination_urls,
     handleSqlError,
     non_match_for_deletion,
@@ -125,20 +127,7 @@ def admin_email():
     form = AdminTestEmailForm()
 
     if "poll" in request.args:  # pragma: no cover
-        try:
-            result = celery.AsyncResult(request.args["poll"])
-            ready = result.ready()
-            return {
-                "ready": ready,
-                "successful": result.successful() if ready else None,
-                "value": result.get() if ready else result.result,
-            }
-        except Exception as e:
-            return {
-                "ready": True,
-                "successful": False,
-                "error": getattr(e, "message", "Unknown error"),
-            }
+        return get_celery_poll_result()
 
     if form.validate_on_submit():
         subject = gettext(
@@ -167,21 +156,7 @@ def admin_newsletter():
     form = AdminNewsletterForm()
 
     if "poll" in request.args:  # pragma: no cover
-        try:
-            result = celery.GroupResult.restore(request.args["poll"])
-            ready = result.ready()
-            return {
-                "ready": ready,
-                "count": len(result.children),
-                "completed": result.completed_count(),
-                "successful": result.successful() if ready else None,
-            }
-        except Exception as e:
-            return {
-                "ready": True,
-                "successful": False,
-                "error": getattr(e, "message", "Unknown error"),
-            }
+        return get_celery_poll_group_result()
 
     if form.validate_on_submit():
         subject = gettext(

project/views/manage.py

@@ -1,10 +1,19 @@
-from flask import flash, redirect, render_template, request, url_for
+import os
+
+from flask import (
+    flash,
+    redirect,
+    render_template,
+    request,
+    send_from_directory,
+    url_for,
+)
 from flask_babel import gettext
 from flask_security import auth_required, current_user
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.sql import desc, func
 
-from project import app, db
+from project import app, db, dump_org_path
 from project.access import (
     access_or_401,
     admin_unit_suggestions_enabled_or_404,
@@ -12,6 +21,7 @@ from project.access import (
     get_admin_units_for_manage,
     has_access,
 )
+from project.celery_tasks import dump_admin_unit_task
 from project.forms.admin_unit import UpdateAdminUnitWidgetForm
 from project.forms.event import FindEventForm
 from project.forms.event_place import FindEventPlaceForm
@@ -35,6 +45,7 @@ from project.utils import get_place_str
 from project.views.event import get_event_category_choices
 from project.views.utils import (
     flash_errors,
+    get_celery_poll_result,
     get_current_admin_unit,
     get_pagination_urls,
     handleSqlError,
@@ -336,6 +347,51 @@ def manage_admin_unit_events_import(id):
     )
 
 
+@app.route("/manage/admin_unit/<int:id>/export", methods=["GET", "POST"])
+@auth_required()
+def manage_admin_unit_export(id):
+    admin_unit = get_admin_unit_for_manage_or_404(id)
+
+    if not has_access(admin_unit, "admin_unit:update"):  # pragma: no cover
+        return permission_missing(url_for("manage_admin_unit", id=admin_unit.id))
+
+    if "poll" in request.args:  # pragma: no cover
+        return get_celery_poll_result()
+
+    if request.method == "POST":  # pragma: no cover
+        result = dump_admin_unit_task.delay(admin_unit.id)
+        return {"result_id": result.id}
+
+    set_current_admin_unit(admin_unit)
+
+    file_name = f"org-{admin_unit.id}.zip"
+    file_path = os.path.join(dump_org_path, file_name)
+    dump_file = None
+
+    if os.path.exists(file_path):
+        dump_file = {
+            "url": url_for(
+                "manage_admin_unit_export_dump_files", id=admin_unit.id, path=file_name
+            ),
+            "size": os.path.getsize(file_path),
+            "ctime": os.path.getctime(file_path),
+        }
+
+    return render_template(
+        "manage/export.html",
+        admin_unit=admin_unit,
+        dump_file=dump_file,
+    )
+
+
+@app.route("/manage/admin_unit/<int:id>/export/dump/<path:path>")
+def manage_admin_unit_export_dump_files(id, path):
+    admin_unit = get_admin_unit_for_manage_or_404(id)
+    access_or_401(admin_unit, "admin_unit:update")
+
+    return send_from_directory(dump_org_path, path)
+
+
 @app.route("/manage/admin_unit/<int:id>/widgets", methods=("GET", "POST"))
 @auth_required()
 def manage_admin_unit_widgets(id):

project/views/utils.py

@@ -9,7 +9,7 @@ from psycopg2.errorcodes import UNIQUE_VIOLATION
 from sqlalchemy.exc import SQLAlchemyError
 from wtforms import FormField
 
-from project import app, mail
+from project import app, celery, mail
 from project.access import get_admin_unit_for_manage, get_admin_units_for_manage
 from project.dateutils import berlin_tz, round_to_next_day
 from project.models import Event, EventAttendanceMode, EventDate
@@ -263,3 +263,38 @@ def get_invitation_access_result(email: str):
         return app.login_manager.unauthorized()
 
     return None
+
+
+def get_celery_poll_result():  # pragma: no cover
+    try:
+        result = celery.AsyncResult(request.args["poll"])
+        ready = result.ready()
+        return {
+            "ready": ready,
+            "successful": result.successful() if ready else None,
+            "value": result.get() if ready else result.result,
+        }
+    except Exception as e:
+        return {
+            "ready": True,
+            "successful": False,
+            "error": getattr(e, "message", "Unknown error"),
+        }
+
+
+def get_celery_poll_group_result():  # pragma: no cover
+    try:
+        result = celery.GroupResult.restore(request.args["poll"])
+        ready = result.ready()
+        return {
+            "ready": ready,
+            "count": len(result.children),
+            "completed": result.completed_count(),
+            "successful": result.successful() if ready else None,
+        }
+    except Exception as e:
+        return {
+            "ready": True,
+            "successful": False,
+            "error": getattr(e, "message", "Unknown error"),
+        }

tests/cli/test_dump.py

@@ -9,3 +9,25 @@ def test_all(client, seeder, app, utils):
 
     utils.get_endpoint_ok("developer")
     utils.get_endpoint_ok("dump_files", path="all.zip")
+
+
+def test_organization(client, seeder, app, utils):
+    user_id, admin_unit_id = seeder.setup_base()
+    seeder.create_event(admin_unit_id)
+
+    runner = app.test_cli_runner()
+    result = runner.invoke(
+        args=[
+            "dump",
+            "organization",
+            str(admin_unit_id),
+        ]
+    )
+    assert result.exit_code == 0
+
+    utils.get_endpoint_ok("manage_admin_unit_export", id=admin_unit_id)
+    utils.get_endpoint_ok(
+        "manage_admin_unit_export_dump_files",
+        id=admin_unit_id,
+        path=f"org-{admin_unit_id}.zip",
+    )

tests/services/test_dump.py

@@ -0,0 +1,19 @@
+def test_clear_admin_unit_dumps(client, seeder, utils, app):
+    user_id, admin_unit_id = seeder.setup_base()
+
+    with app.app_context():
+        from project.services.dump import clear_admin_unit_dumps
+
+        clear_admin_unit_dumps()
+
+
+def test_dump_admin_unit(client, seeder, utils, app):
+    user_id, admin_unit_id = seeder.setup_base()
+    event_id = seeder.create_event(admin_unit_id)
+    image_id = seeder.upsert_default_image()
+    seeder.assign_image_to_event(event_id, image_id)
+
+    with app.app_context():
+        from project.services.dump import dump_admin_unit
+
+        dump_admin_unit(admin_unit_id)