diff --git a/project/api/image/schemas.py b/project/api/image/schemas.py index 8006bd8..f74028d 100644 --- a/project/api/image/schemas.py +++ b/project/api/image/schemas.py @@ -1,7 +1,7 @@ from marshmallow import ValidationError, fields, post_load, validate, validates_schema from project.api import marshmallow -from project.api.schemas import SQLAlchemyBaseSchema +from project.api.schemas import IdSchemaMixin, SQLAlchemyBaseSchema from project.imageutils import ( get_bytes_from_image, get_image_from_base64_str, @@ -36,7 +36,7 @@ class ImageSchema(ImageModelSchema, ImageBaseSchemaMixin): return url_for_image(image) -class ImageDumpSchema(ImageModelSchema, ImageBaseSchemaMixin): +class ImageDumpSchema(ImageModelSchema, IdSchemaMixin, ImageBaseSchemaMixin): pass diff --git a/project/celery_tasks.py b/project/celery_tasks.py index 01d0b7c..384f04f 100644 --- a/project/celery_tasks.py +++ b/project/celery_tasks.py @@ -42,6 +42,16 @@ def dump_all_task(): dump_all() +@celery.task( + acks_late=True, + reject_on_worker_lost=True, +) +def dump_admin_unit_task(admin_unit_id): + from project.services.dump import dump_admin_unit + + dump_admin_unit(admin_unit_id) + + @celery.task( acks_late=True, reject_on_worker_lost=True, diff --git a/project/models/image.py b/project/models/image.py index 757d221..cf773a9 100644 --- a/project/models/image.py +++ b/project/models/image.py @@ -22,3 +22,6 @@ class Image(db.Model, TrackableMixin): if self.updated_at else 0 ) + + def get_file_extension(self): + return self.encoding_format.split("/")[-1] if self.encoding_format else "png" diff --git a/project/services/dump.py b/project/services/dump.py index 2e197f7..0381b8a 100644 --- a/project/services/dump.py +++ b/project/services/dump.py @@ -12,6 +12,7 @@ from project.api.event_reference.schemas import EventReferenceDumpSchema from project.api.organization.schemas import OrganizationDumpSchema from project.api.organizer.schemas import OrganizerDumpSchema from project.api.place.schemas import PlaceDumpSchema +from project.imageutils import get_image_from_bytes from project.models import ( AdminUnit, Event, @@ -24,71 +25,154 @@ from project.models import ( from project.utils import make_dir -def dump_items(items, schema, file_base_name, dump_path): - result = schema.dump(items) - path = os.path.join(dump_path, file_base_name + ".json") +class Dumper(object): + def __init__(self, dump_path, file_base_name): + self.dump_path = dump_path + self.file_base_name = file_base_name + self.tmp_path = None - with open(path, "w") as outfile: - json.dump(result, outfile, ensure_ascii=False) + def dump(self): + self.setup_tmp_dir() + self.dump_data() + self.zip_tmp_dir() + self.clean_up_tmp_dir() - app.logger.info(f"{len(items)} item(s) dumped to {path}.") + def dump_data(self): + # Events + events = ( + Event.query.join(Event.admin_unit) + .options(joinedload(Event.categories)) + .filter( + and_( + Event.public_status == PublicStatus.published, + AdminUnit.is_verified, + ) + ) + .all() + ) + self.dump_items(events, EventDumpSchema(many=True), "events") + + # Places + places = EventPlace.query.all() + self.dump_items(places, PlaceDumpSchema(many=True), "places") + + # Event categories + event_categories = EventCategory.query.all() + self.dump_items( + event_categories, + EventCategoryDumpSchema(many=True), + "event_categories", + ) + + # Organizers + organizers = EventOrganizer.query.all() + self.dump_items(organizers, OrganizerDumpSchema(many=True), "organizers") + + # Organizations + organizations = AdminUnit.query.all() + self.dump_items( + organizations, OrganizationDumpSchema(many=True), "organizations" + ) + + # Event references + event_references = EventReference.query.all() + self.dump_items( + event_references, + EventReferenceDumpSchema(many=True), + "event_references", + ) + + def dump_items(self, items, schema, file_base_name): + result = schema.dump(items) + path = os.path.join(self.tmp_path, file_base_name + ".json") + + with open(path, "w") as outfile: + json.dump(result, outfile, ensure_ascii=False, indent=4) + + app.logger.info(f"{len(items)} item(s) dumped to {path}.") + + def dump_item(self, items, schema, file_base_name): # pragma: no cover + result = schema.dump(items) + path = os.path.join(self.tmp_path, file_base_name + ".json") + + with open(path, "w") as outfile: + json.dump(result, outfile, ensure_ascii=False, indent=4) + + app.logger.info(f"Item dumped to {path}.") + + def setup_tmp_dir(self): + self.tmp_path = os.path.join(self.dump_path, f"tmp-{self.file_base_name}") + make_dir(self.tmp_path) + + def clean_up_tmp_dir(self): + shutil.rmtree(self.tmp_path, ignore_errors=True) + + def zip_tmp_dir(self): + zip_base_name = os.path.join(dump_path, self.file_base_name) + zip_path = shutil.make_archive(zip_base_name, "zip", self.tmp_path) + app.logger.info(f"Zipped all up to {zip_path}.") + + def dump_image(self, image): # pragma: no cover + if not image: + return + + extension = image.get_file_extension() + file_path = os.path.join(self.tmp_path, f"{image.id}.{extension}") + get_image_from_bytes(image.data).save(file_path) + + +class AdminUnitDumper(Dumper): # pragma: no cover + def __init__(self, dump_path, admin_unit_id): + super().__init__(dump_path, f"org-{admin_unit_id}") + self.admin_unit_id = admin_unit_id + + def dump_data(self): + # Events + events = ( + Event.query.join(Event.admin_unit) + .options(joinedload(Event.categories)) + .filter(Event.admin_unit_id == self.admin_unit_id) + .all() + ) + self.dump_items(events, EventDumpSchema(many=True), "events") + for event in events: + self.dump_image(event.photo) + + # Places + places = EventPlace.query.filter( + EventPlace.admin_unit_id == self.admin_unit_id + ).all() + self.dump_items(places, PlaceDumpSchema(many=True), "places") + for place in places: + self.dump_image(place.photo) + + # Event categories + event_categories = EventCategory.query.all() + self.dump_items( + event_categories, + EventCategoryDumpSchema(many=True), + "event_categories", + ) + + # Organizers + organizers = EventOrganizer.query.filter( + EventOrganizer.admin_unit_id == self.admin_unit_id + ).all() + self.dump_items(organizers, OrganizerDumpSchema(many=True), "organizers") + for organizer in organizers: + self.dump_image(organizer.logo) + + # Organizations + organization = AdminUnit.query.get(self.admin_unit_id) + self.dump_item(organization, OrganizationDumpSchema(), "organization") + self.dump_image(organization.logo) def dump_all(): - # Setup temp dir - tmp_path = os.path.join(dump_path, "tmp") - make_dir(tmp_path) + dumper = Dumper(dump_path, "all") + dumper.dump() - # Events - events = ( - Event.query.join(Event.admin_unit) - .options(joinedload(Event.categories)) - .filter( - and_( - Event.public_status == PublicStatus.published, - AdminUnit.is_verified, - ) - ) - .all() - ) - dump_items(events, EventDumpSchema(many=True), "events", tmp_path) - # Places - places = EventPlace.query.all() - dump_items(places, PlaceDumpSchema(many=True), "places", tmp_path) - - # Event categories - event_categories = EventCategory.query.all() - dump_items( - event_categories, - EventCategoryDumpSchema(many=True), - "event_categories", - tmp_path, - ) - - # Organizers - organizers = EventOrganizer.query.all() - dump_items(organizers, OrganizerDumpSchema(many=True), "organizers", tmp_path) - - # Organizations - organizations = AdminUnit.query.all() - dump_items( - organizations, OrganizationDumpSchema(many=True), "organizations", tmp_path - ) - - # Event references - event_references = EventReference.query.all() - dump_items( - event_references, - EventReferenceDumpSchema(many=True), - "event_references", - tmp_path, - ) - - # Zip - zip_base_name = os.path.join(dump_path, "all") - zip_path = shutil.make_archive(zip_base_name, "zip", tmp_path) - app.logger.info(f"Zipped all up to {zip_path}.") - - # Clean up temp dir - shutil.rmtree(tmp_path, ignore_errors=True) +def dump_admin_unit(admin_unit_id): # pragma: no cover + dumper = AdminUnitDumper(dump_path, admin_unit_id) + dumper.dump() diff --git a/project/views/image.py b/project/views/image.py index f025517..2a3d247 100644 --- a/project/views/image.py +++ b/project/views/image.py @@ -1,11 +1,11 @@ import os -from io import BytesIO import PIL from flask import request, send_file from sqlalchemy.orm import load_only from project import app, img_path +from project.imageutils import get_image_from_bytes from project.models import Image from project.utils import make_dir @@ -26,7 +26,7 @@ def image(id, hash=None): height = width # Generate file name - extension = image.encoding_format.split("/")[-1] if image.encoding_format else "png" + extension = image.get_file_extension() hash = image.get_hash() file_path = os.path.join(img_path, f"{id}-{hash}-{width}-{height}.{extension}") @@ -36,7 +36,7 @@ def image(id, hash=None): # Save from database to disk make_dir(img_path) - img = PIL.Image.open(BytesIO(image.data)) + img = get_image_from_bytes(image.data) img.thumbnail((width, height), PIL.Image.ANTIALIAS) img.save(file_path)