Merge pull request #425 from eventcally/issues/424

Export for organization #424

project/api/image/schemas.py

@@ -1,7 +1,7 @@
 from marshmallow import ValidationError, fields, post_load, validate, validates_schema
 
 from project.api import marshmallow
-from project.api.schemas import SQLAlchemyBaseSchema
+from project.api.schemas import IdSchemaMixin, SQLAlchemyBaseSchema
 from project.imageutils import (
     get_bytes_from_image,
     get_image_from_base64_str,
@@ -36,7 +36,7 @@ class ImageSchema(ImageModelSchema, ImageBaseSchemaMixin):
         return url_for_image(image)
 
 
-class ImageDumpSchema(ImageModelSchema, ImageBaseSchemaMixin):
+class ImageDumpSchema(ImageModelSchema, IdSchemaMixin, ImageBaseSchemaMixin):
     pass
 
 

project/celery_tasks.py

@@ -42,6 +42,16 @@ def dump_all_task():
     dump_all()
 
 
+@celery.task(
+    acks_late=True,
+    reject_on_worker_lost=True,
+)
+def dump_admin_unit_task(admin_unit_id):
+    from project.services.dump import dump_admin_unit
+
+    dump_admin_unit(admin_unit_id)
+
+
 @celery.task(
     acks_late=True,
     reject_on_worker_lost=True,

project/models/image.py

@@ -22,3 +22,6 @@ class Image(db.Model, TrackableMixin):
             if self.updated_at
             else 0
         )
+
+    def get_file_extension(self):
+        return self.encoding_format.split("/")[-1] if self.encoding_format else "png"

project/services/dump.py

@@ -12,6 +12,7 @@ from project.api.event_reference.schemas import EventReferenceDumpSchema
 from project.api.organization.schemas import OrganizationDumpSchema
 from project.api.organizer.schemas import OrganizerDumpSchema
 from project.api.place.schemas import PlaceDumpSchema
+from project.imageutils import get_image_from_bytes
 from project.models import (
     AdminUnit,
     Event,
@@ -24,71 +25,154 @@ from project.models import (
 from project.utils import make_dir
 
 
-def dump_items(items, schema, file_base_name, dump_path):
-    result = schema.dump(items)
-    path = os.path.join(dump_path, file_base_name + ".json")
+class Dumper(object):
+    def __init__(self, dump_path, file_base_name):
+        self.dump_path = dump_path
+        self.file_base_name = file_base_name
+        self.tmp_path = None
 
-    with open(path, "w") as outfile:
-        json.dump(result, outfile, ensure_ascii=False)
+    def dump(self):
+        self.setup_tmp_dir()
+        self.dump_data()
+        self.zip_tmp_dir()
+        self.clean_up_tmp_dir()
 
-    app.logger.info(f"{len(items)} item(s) dumped to {path}.")
+    def dump_data(self):
+        # Events
+        events = (
+            Event.query.join(Event.admin_unit)
+            .options(joinedload(Event.categories))
+            .filter(
+                and_(
+                    Event.public_status == PublicStatus.published,
+                    AdminUnit.is_verified,
+                )
+            )
+            .all()
+        )
+        self.dump_items(events, EventDumpSchema(many=True), "events")
+
+        # Places
+        places = EventPlace.query.all()
+        self.dump_items(places, PlaceDumpSchema(many=True), "places")
+
+        # Event categories
+        event_categories = EventCategory.query.all()
+        self.dump_items(
+            event_categories,
+            EventCategoryDumpSchema(many=True),
+            "event_categories",
+        )
+
+        # Organizers
+        organizers = EventOrganizer.query.all()
+        self.dump_items(organizers, OrganizerDumpSchema(many=True), "organizers")
+
+        # Organizations
+        organizations = AdminUnit.query.all()
+        self.dump_items(
+            organizations, OrganizationDumpSchema(many=True), "organizations"
+        )
+
+        # Event references
+        event_references = EventReference.query.all()
+        self.dump_items(
+            event_references,
+            EventReferenceDumpSchema(many=True),
+            "event_references",
+        )
+
+    def dump_items(self, items, schema, file_base_name):
+        result = schema.dump(items)
+        path = os.path.join(self.tmp_path, file_base_name + ".json")
+
+        with open(path, "w") as outfile:
+            json.dump(result, outfile, ensure_ascii=False, indent=4)
+
+        app.logger.info(f"{len(items)} item(s) dumped to {path}.")
+
+    def dump_item(self, items, schema, file_base_name):  # pragma: no cover
+        result = schema.dump(items)
+        path = os.path.join(self.tmp_path, file_base_name + ".json")
+
+        with open(path, "w") as outfile:
+            json.dump(result, outfile, ensure_ascii=False, indent=4)
+
+        app.logger.info(f"Item dumped to {path}.")
+
+    def setup_tmp_dir(self):
+        self.tmp_path = os.path.join(self.dump_path, f"tmp-{self.file_base_name}")
+        make_dir(self.tmp_path)
+
+    def clean_up_tmp_dir(self):
+        shutil.rmtree(self.tmp_path, ignore_errors=True)
+
+    def zip_tmp_dir(self):
+        zip_base_name = os.path.join(dump_path, self.file_base_name)
+        zip_path = shutil.make_archive(zip_base_name, "zip", self.tmp_path)
+        app.logger.info(f"Zipped all up to {zip_path}.")
+
+    def dump_image(self, image):  # pragma: no cover
+        if not image:
+            return
+
+        extension = image.get_file_extension()
+        file_path = os.path.join(self.tmp_path, f"{image.id}.{extension}")
+        get_image_from_bytes(image.data).save(file_path)
+
+
+class AdminUnitDumper(Dumper):  # pragma: no cover
+    def __init__(self, dump_path, admin_unit_id):
+        super().__init__(dump_path, f"org-{admin_unit_id}")
+        self.admin_unit_id = admin_unit_id
+
+    def dump_data(self):
+        # Events
+        events = (
+            Event.query.join(Event.admin_unit)
+            .options(joinedload(Event.categories))
+            .filter(Event.admin_unit_id == self.admin_unit_id)
+            .all()
+        )
+        self.dump_items(events, EventDumpSchema(many=True), "events")
+        for event in events:
+            self.dump_image(event.photo)
+
+        # Places
+        places = EventPlace.query.filter(
+            EventPlace.admin_unit_id == self.admin_unit_id
+        ).all()
+        self.dump_items(places, PlaceDumpSchema(many=True), "places")
+        for place in places:
+            self.dump_image(place.photo)
+
+        # Event categories
+        event_categories = EventCategory.query.all()
+        self.dump_items(
+            event_categories,
+            EventCategoryDumpSchema(many=True),
+            "event_categories",
+        )
+
+        # Organizers
+        organizers = EventOrganizer.query.filter(
+            EventOrganizer.admin_unit_id == self.admin_unit_id
+        ).all()
+        self.dump_items(organizers, OrganizerDumpSchema(many=True), "organizers")
+        for organizer in organizers:
+            self.dump_image(organizer.logo)
+
+        # Organizations
+        organization = AdminUnit.query.get(self.admin_unit_id)
+        self.dump_item(organization, OrganizationDumpSchema(), "organization")
+        self.dump_image(organization.logo)
 
 
 def dump_all():
-    # Setup temp dir
-    tmp_path = os.path.join(dump_path, "tmp")
-    make_dir(tmp_path)
+    dumper = Dumper(dump_path, "all")
+    dumper.dump()
 
-    # Events
-    events = (
-        Event.query.join(Event.admin_unit)
-        .options(joinedload(Event.categories))
-        .filter(
-            and_(
-                Event.public_status == PublicStatus.published,
-                AdminUnit.is_verified,
-            )
-        )
-        .all()
-    )
-    dump_items(events, EventDumpSchema(many=True), "events", tmp_path)
 
-    # Places
-    places = EventPlace.query.all()
-    dump_items(places, PlaceDumpSchema(many=True), "places", tmp_path)
-
-    # Event categories
-    event_categories = EventCategory.query.all()
-    dump_items(
-        event_categories,
-        EventCategoryDumpSchema(many=True),
-        "event_categories",
-        tmp_path,
-    )
-
-    # Organizers
-    organizers = EventOrganizer.query.all()
-    dump_items(organizers, OrganizerDumpSchema(many=True), "organizers", tmp_path)
-
-    # Organizations
-    organizations = AdminUnit.query.all()
-    dump_items(
-        organizations, OrganizationDumpSchema(many=True), "organizations", tmp_path
-    )
-
-    # Event references
-    event_references = EventReference.query.all()
-    dump_items(
-        event_references,
-        EventReferenceDumpSchema(many=True),
-        "event_references",
-        tmp_path,
-    )
-
-    # Zip
-    zip_base_name = os.path.join(dump_path, "all")
-    zip_path = shutil.make_archive(zip_base_name, "zip", tmp_path)
-    app.logger.info(f"Zipped all up to {zip_path}.")
-
-    # Clean up temp dir
-    shutil.rmtree(tmp_path, ignore_errors=True)
+def dump_admin_unit(admin_unit_id):  # pragma: no cover
+    dumper = AdminUnitDumper(dump_path, admin_unit_id)
+    dumper.dump()

project/views/image.py

@@ -1,11 +1,11 @@
 import os
-from io import BytesIO
 
 import PIL
 from flask import request, send_file
 from sqlalchemy.orm import load_only
 
 from project import app, img_path
+from project.imageutils import get_image_from_bytes
 from project.models import Image
 from project.utils import make_dir
 
@@ -26,7 +26,7 @@ def image(id, hash=None):
         height = width
 
     # Generate file name
-    extension = image.encoding_format.split("/")[-1] if image.encoding_format else "png"
+    extension = image.get_file_extension()
     hash = image.get_hash()
     file_path = os.path.join(img_path, f"{id}-{hash}-{width}-{height}.{extension}")
 
@@ -36,7 +36,7 @@ def image(id, hash=None):
 
     # Save from database to disk
     make_dir(img_path)
-    img = PIL.Image.open(BytesIO(image.data))
+    img = get_image_from_bytes(image.data)
     img.thumbnail((width, height), PIL.Image.ANTIALIAS)
     img.save(file_path)