Send notification mails via celery #509

project/__init__.py

@@ -197,7 +197,7 @@ db = SQLAlchemy(app, metadata=metadata)
 migrate = Migrate(app, db, render_as_batch=False)
 
 # Celery tasks
-from project import celery_tasks
+from project import base_tasks, celery_tasks
 
 # API
 from project.api import RestApi

project/api/organization/resources.py

@@ -95,7 +95,7 @@ from project.services.reference import (
     get_reference_outgoing_query,
     get_relation_outgoing_query,
 )
-from project.views.utils import get_current_admin_unit_for_api, send_mail
+from project.views.utils import get_current_admin_unit_for_api, send_mail_async
 
 
 class OrganizationResource(BaseResource):
@@ -451,7 +451,7 @@ class OrganizationOrganizationInvitationListResource(BaseResource):
         db.session.add(invitation)
         db.session.commit()
 
-        send_mail(
+        send_mail_async(
             invitation.email,
             gettext("You have received an invitation"),
             "organization_invitation_notice",

project/base_tasks.py

@@ -1,12 +1,12 @@
 from project import app, celery
 from project.celery import force_locale
-from project.views.utils import send_mail
+from project.views.utils import send_mails_with_body
 
 
 @celery.task(
     base=getattr(app, "celery_http_task_cls"),
     priority=0,
 )
-def send_mail_task(recipient, subject, template, **context):
+def send_mail_with_body_task(recipient, subject, body, html):
     with force_locale():
-        send_mail(recipient, subject, template, **context)
+        send_mails_with_body([recipient], subject, body, html)

project/views/admin.py

@@ -1,4 +1,3 @@
-from celery import group
 from flask import flash, redirect, render_template, request, url_for
 from flask_babel import gettext
 from flask_security import roles_required
@@ -6,7 +5,6 @@ from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.sql import func
 
 from project import app, db
-from project.base_tasks import send_mail_task
 from project.forms.admin import (
     AdminNewsletterForm,
     AdminSettingsForm,
@@ -24,11 +22,12 @@ from project.services.user import delete_user, set_roles_for_user
 from project.views.utils import (
     flash_errors,
     get_celery_poll_group_result,
-    get_celery_poll_result,
     get_pagination_urls,
     handleSqlError,
     non_match_for_deletion,
     send_mail,
+    send_mail_async,
+    send_mails_async,
 )
 
 
@@ -148,7 +147,7 @@ def admin_email():
     form = AdminTestEmailForm()
 
     if "poll" in request.args:  # pragma: no cover
-        return get_celery_poll_result()
+        return get_celery_poll_group_result()
 
     if form.validate_on_submit():
         subject = gettext(
@@ -157,7 +156,8 @@ def admin_email():
         )
 
         if "async" in request.args:  # pragma: no cover
-            result = send_mail_task.delay(form.recipient.data, subject, "test_email")
+            result = send_mail_async(form.recipient.data, subject, "test_email")
+            result.save()
             return {"result_id": result.id}
 
         try:
@@ -196,10 +196,9 @@ def admin_newsletter():
             )
             recipients = [u.email for u in users]
 
-        result = group(
+        result = send_mails_async(
-            send_mail_task.s(r, subject, "newsletter", message=form.message.data)
+            recipients, subject, "newsletter", message=form.message.data
-            for r in recipients
+        )
-        ).delay()
         result.save()
         return {"result_id": result.id}
 

project/views/admin_unit.py

@@ -32,7 +32,7 @@ from project.views.utils import (
     manage_required,
     non_match_for_deletion,
     permission_missing,
-    send_mails,
+    send_mails_async,
 )
 
 
@@ -260,7 +260,7 @@ def send_admin_unit_invitation_accepted_mails(
     )
     emails = list(map(lambda member: member.user.email, members))
 
-    send_mails(
+    send_mails_async(
         emails,
         gettext("Organization invitation accepted"),
         "organization_invitation_accepted_notice",
@@ -274,7 +274,7 @@ def send_admin_unit_deletion_requested_mails(admin_unit: AdminUnit):
     members = get_admin_unit_members_with_permission(admin_unit.id, "admin_unit:update")
     emails = list(map(lambda member: member.user.email, members))
 
-    send_mails(
+    send_mails_async(
         emails,
         gettext("Organization deletion requested"),
         "organization_deletion_requested_notice",

project/views/admin_unit_member_invitation.py

@@ -18,7 +18,7 @@ from project.views.utils import (
     handleSqlError,
     non_match_for_deletion,
     permission_missing,
-    send_mail,
+    send_mail_async,
 )
 
 
@@ -84,7 +84,7 @@ def manage_admin_unit_member_invite(id):
             db.session.add(invitation)
             db.session.commit()
 
-            send_mail(
+            send_mail_async(
                 invitation.email,
                 gettext("You have received an invitation"),
                 "invitation_notice",

project/views/event.py

@@ -52,7 +52,7 @@ from project.views.utils import (
     get_calendar_links_for_event,
     get_share_links,
     handleSqlError,
-    send_mails,
+    send_mails_async,
     set_current_admin_unit,
 )
 
@@ -452,7 +452,7 @@ def send_referenced_event_changed_mails(event):
         )
         emails = list(map(lambda member: member.user.email, members))
 
-        send_mails(
+        send_mails_async(
             emails,
             gettext("Referenced event changed"),
             "referenced_event_changed_notice",
@@ -475,7 +475,7 @@ def send_event_report_mails(event: Event, report: dict):
     admin_emails = list(map(lambda admin: admin.email, admins))
     emails.extend(x for x in admin_emails if x not in emails)
 
-    send_mails(
+    send_mails_async(
         emails,
         gettext("New event report"),
         "event_report_notice",

project/views/event_suggestion.py

@@ -7,7 +7,7 @@ from project import app, db
 from project.access import access_or_401
 from project.forms.event_suggestion import RejectEventSuggestionForm
 from project.models import EventReviewStatus, EventSuggestion
-from project.views.utils import flash_errors, handleSqlError, send_mail
+from project.views.utils import flash_errors, handleSqlError, send_mail_async
 
 
 @app.route("/event_suggestion/<int:event_suggestion_id>/review")
@@ -82,7 +82,7 @@ def event_suggestion_review_status(event_suggestion_id):
 
 def send_event_suggestion_review_status_mail(event_suggestion):
     if event_suggestion.contact_email and event_suggestion.contact_email_notice:
-        send_mail(
+        send_mail_async(
             event_suggestion.contact_email,
             gettext("Event review status updated"),
             "review_status_notice",

project/views/reference_request.py

@@ -29,7 +29,7 @@ from project.views.utils import (
     flash_errors,
     get_pagination_urls,
     handleSqlError,
-    send_mails,
+    send_mails_async,
 )
 
 
@@ -159,7 +159,7 @@ def send_member_reference_request_verify_mails(
     )
     emails = list(map(lambda member: member.user.email, members))
 
-    send_mails(emails, subject, template, **context)
+    send_mails_async(emails, subject, template, **context)
 
 
 def send_reference_request_inbox_mails(request):

project/views/reference_request_review.py

@@ -21,7 +21,7 @@ from project.services.admin_unit import (
     upsert_admin_unit_relation,
 )
 from project.services.reference import create_event_reference_for_request
-from project.views.utils import flash_errors, handleSqlError, send_mails
+from project.views.utils import flash_errors, handleSqlError, send_mails_async
 
 
 @app.route("/reference_request/<int:id>/review", methods=("GET", "POST"))
@@ -130,7 +130,7 @@ def send_reference_request_review_status_mails(request):
     )
     emails = list(map(lambda member: member.user.email, members))
 
-    send_mails(
+    send_mails_async(
         emails,
         gettext("Event review status updated"),
         "reference_request_review_status_notice",

project/views/user.py

@@ -20,7 +20,7 @@ from project.views.utils import (
     get_invitation_access_result,
     handleSqlError,
     non_match_for_deletion,
-    send_mail,
+    send_mail_async,
 )
 
 
@@ -155,7 +155,7 @@ def user_cancel_deletion():
 
 
 def send_user_deletion_requested_mail(user):
-    send_mail(
+    send_mail_async(
         user.email,
         gettext("User deletion requested"),
         "user_deletion_requested_notice",

project/views/utils.py

@@ -168,9 +168,37 @@ def send_mails(recipients, subject, template, **context):
     if len(recipients) == 0:  # pragma: no cover
         return
 
+    body, html = render_mail_body(template, **context)
+    send_mails_with_body(recipients, subject, body, html)
+
+
+def send_mail_async(recipient, subject, template, **context):
+    send_mails_async([recipient], subject, template, **context)
+
+
+def send_mails_async(recipients, subject, template, **context):
+    if len(recipients) == 0:  # pragma: no cover
+        return
+
+    body, html = render_mail_body(template, **context)
+    return send_mails_with_body_async(recipients, subject, body, html)
+
+
+def send_mails_with_body_async(recipients, subject, body, html):
+    from celery import group
+
+    from project.base_tasks import send_mail_with_body_task
+
+    result = group(
+        send_mail_with_body_task.s(r, subject, body, html) for r in recipients
+    ).delay()
+    return result
+
+
+def render_mail_body(template, **context):
     body = render_template("email/%s.txt" % template, **context)
     html = render_template("email/%s.html" % template, **context)
-    send_mails_with_body(recipients, subject, body, html)
+    return body, html
 
 
 def send_mails_with_body(recipients, subject, body, html):

project/views/verification_request.py

@@ -22,7 +22,7 @@ from project.views.utils import (
     handleSqlError,
     manage_required,
     non_match_for_deletion,
-    send_mails,
+    send_mails_async,
 )
 
 
@@ -192,7 +192,7 @@ def send_member_verification_request_verify_mails(
     )
     emails = list(map(lambda member: member.user.email, members))
 
-    send_mails(emails, subject, template, **context)
+    send_mails_async(emails, subject, template, **context)
 
 
 def send_verification_request_inbox_mails(request):

project/views/verification_request_review.py

@@ -15,7 +15,7 @@ from project.models import (
     AdminUnitVerificationRequestReviewStatus,
 )
 from project.services.admin_unit import upsert_admin_unit_relation
-from project.views.utils import flash_errors, handleSqlError, send_mails
+from project.views.utils import flash_errors, handleSqlError, send_mails_async
 
 
 @app.route("/verification_request/<int:id>/review", methods=("GET", "POST"))
@@ -110,7 +110,7 @@ def send_verification_request_review_status_mails(request):
     )
     emails = list(map(lambda member: member.user.email, members))
 
-    send_mails(
+    send_mails_async(
         emails,
         gettext("Verification request review status updated"),
         "verification_request_review_status_notice",

project/views/widget.py

@@ -30,7 +30,7 @@ from project.views.utils import (
     get_pagination_urls,
     get_share_links,
     handleSqlError,
-    send_mails,
+    send_mails_async,
 )
 
 
@@ -214,7 +214,7 @@ def send_event_inbox_mails(admin_unit, event_suggestion):
     members = get_admin_unit_members_with_permission(admin_unit.id, "event:verify")
     emails = list(map(lambda member: member.user.email, members))
 
-    send_mails(
+    send_mails_async(
         emails,
         gettext("New event review"),
         "review_notice",

tests/api/test_event.py

@@ -542,7 +542,7 @@ def test_put_referencedEventUpdate_sendsMail(
     other_admin_unit_id = seeder.create_admin_unit(other_user_id, "Other Crew")
     seeder.create_reference(event_id, other_admin_unit_id)
 
-    mail_mock = utils.mock_send_mails(mocker)
+    mail_mock = utils.mock_send_mails_async(mocker)
     url = utils.get_url("api_v1_event", id=event_id)
     put = create_put(place_id, organizer_id)
     put["name"] = "Changed name"
@@ -564,7 +564,7 @@ def test_put_referencedEventNonDirtyUpdate_doesNotSendMail(
     other_admin_unit_id = seeder.create_admin_unit(other_user_id, "Other Crew")
     seeder.create_reference(event_id, other_admin_unit_id)
 
-    mail_mock = utils.mock_send_mails(mocker)
+    mail_mock = utils.mock_send_mails_async(mocker)
     url = utils.get_url("api_v1_event", id=event_id)
     put = create_put(place_id, organizer_id)
     put["name"] = "Name"
@@ -617,7 +617,7 @@ def test_patch_referencedEventUpdate_sendsMail(
     other_admin_unit_id = seeder.create_admin_unit(other_user_id, "Other Crew")
     seeder.create_reference(event_id, other_admin_unit_id)
 
-    mail_mock = utils.mock_send_mails(mocker)
+    mail_mock = utils.mock_send_mails_async(mocker)
     url = utils.get_url("api_v1_event", id=event_id)
     response = utils.patch_json(url, {"name": "Changed name"})
 
@@ -716,7 +716,7 @@ def test_report_mail(client, seeder: Seeder, utils: UtilActions, app, mocker):
     seeder.create_user(email="admin@test.de", admin=True)
     seeder.create_user(email="normal@test.de", admin=False)
 
-    mail_mock = utils.mock_send_mails(mocker)
+    mail_mock = utils.mock_send_mails_async(mocker)
     url = utils.get_url("api_v1_event_reports", id=event_id)
     response = utils.post_json(
         url,

tests/api/test_organization.py

@@ -625,7 +625,7 @@ def test_organization_invitation_list(client, seeder: Seeder, utils: UtilActions
 
 
 def test_organization_invitation_list_post(client, app, seeder, db, utils, mocker):
-    mail_mock = utils.mock_send_mails(mocker)
+    mail_mock = utils.mock_send_mails_async(mocker)
     _, admin_unit_id = seeder.setup_api_access()
 
     url = utils.get_url(

tests/utils.py

@@ -204,6 +204,9 @@ class UtilActions(object):
     def mock_send_mails(self, mocker):
         return mocker.patch("project.views.utils.send_mails_with_body")
 
+    def mock_send_mails_async(self, mocker):
+        return mocker.patch("project.views.utils.send_mails_with_body_async")
+
     def assert_send_mail_called(
         self, mock, expected_recipients, expected_contents=None
     ):

tests/views/test_admin_unit.py

@@ -138,7 +138,7 @@ def test_create_requiresAdmin_memberOfOrgWithFlag(client, app, utils, seeder):
 
 
 def test_create_from_invitation(client, app, db, utils, seeder, mocker):
-    mail_mock = utils.mock_send_mails(mocker)
+    mail_mock = utils.mock_send_mails_async(mocker)
     user_id = seeder.create_user()
     admin_unit_id = seeder.create_admin_unit(
         user_id, can_invite_other=True, can_verify_other=True

tests/views/test_admin_unit_member_invitation.py

@@ -11,7 +11,7 @@ def test_create(client, app, utils, seeder, mocker):
     assert response.status_code == 200
 
     with client:
-        mail_mock = utils.mock_send_mails(mocker)
+        mail_mock = utils.mock_send_mails_async(mocker)
         email = "new@member.de"
         response = client.post(
             url,

tests/views/test_event_suggestion.py

@@ -50,7 +50,7 @@ def test_reject(client, app, utils, seeder, mocker, db, db_error, is_verified):
     if db_error:
         utils.mock_db_commit(mocker)
 
-    mail_mock = utils.mock_send_mails(mocker)
+    mail_mock = utils.mock_send_mails_async(mocker)
     response = utils.post_form(
         url,
         response,

tests/views/test_reference.py

@@ -197,7 +197,7 @@ def test_referencedEventUpdate_sendsMail(client, seeder, utils, app, mocker):
     url = utils.get_url("event_update", event_id=event_id)
     response = utils.get_ok(url)
 
-    mail_mock = utils.mock_send_mails(mocker)
+    mail_mock = utils.mock_send_mails_async(mocker)
     response = utils.post_form(
         url,
         response,
@@ -226,7 +226,7 @@ def test_referencedEventNonDirtyUpdate_doesNotSendMail(
     url = utils.get_url("event_update", event_id=event_id)
     response = utils.get_ok(url)
 
-    mail_mock = utils.mock_send_mails(mocker)
+    mail_mock = utils.mock_send_mails_async(mocker)
     response = utils.post_form(
         url,
         response,

tests/views/test_reference_request.py

@@ -1,8 +1,11 @@
 import pytest
 
+from tests.seeder import Seeder
+from tests.utils import UtilActions
+
 
 @pytest.mark.parametrize("db_error", [True, False])
-def test_create(client, app, utils, seeder, mocker, db_error):
+def test_create(client, app, utils: UtilActions, seeder: Seeder, mocker, db_error):
     user_id, admin_unit_id = seeder.setup_base()
     event_id = seeder.create_event(admin_unit_id)
 
@@ -15,7 +18,7 @@ def test_create(client, app, utils, seeder, mocker, db_error):
     if db_error:
         utils.mock_db_commit(mocker)
 
-    mail_mock = utils.mock_send_mails(mocker)
+    mail_mock = utils.mock_send_mails_async(mocker)
     response = utils.post_form(
         url,
         response,
@@ -50,7 +53,7 @@ def test_create(client, app, utils, seeder, mocker, db_error):
         )
 
 
-def test_create_duplicateNotAllowed(client, app, utils, seeder):
+def test_create_duplicateNotAllowed(client, app, utils: UtilActions, seeder: Seeder):
     user_id, admin_unit_id = seeder.setup_base()
     event_id = seeder.create_event(admin_unit_id)
     other_user_id = seeder.create_user("other@test.de")
@@ -80,7 +83,9 @@ def test_create_duplicateNotAllowed(client, app, utils, seeder):
     assert b"duplicate key" in response.data
 
 
-def test_create_unverifiedAdminUnitNotAllowed(client, app, utils, seeder):
+def test_create_unverifiedAdminUnitNotAllowed(
+    client, app, utils: UtilActions, seeder: Seeder
+):
     _, admin_unit_id = seeder.setup_base(admin_unit_verified=False)
     event_id = seeder.create_event(admin_unit_id)
     other_user_id = seeder.create_user("other@test.de")
@@ -91,14 +96,14 @@ def test_create_unverifiedAdminUnitNotAllowed(client, app, utils, seeder):
     utils.assert_response_unauthorized(response)
 
 
-def test_create_autoVerify(client, app, utils, seeder, mocker):
+def test_create_autoVerify(client, app, utils: UtilActions, seeder: Seeder, mocker):
     user_id, admin_unit_id = seeder.setup_base()
     event_id = seeder.create_event(admin_unit_id)
     other_user_id = seeder.create_user("other@test.de")
     other_admin_unit_id = seeder.create_admin_unit(other_user_id, "Other Crew")
     seeder.create_admin_unit_relation(other_admin_unit_id, admin_unit_id, True)
 
-    mail_mock = utils.mock_send_mails(mocker)
+    mail_mock = utils.mock_send_mails_async(mocker)
     url = utils.get_url("event_reference_request_create", event_id=event_id)
     response = utils.get_ok(url)
     response = utils.post_form(

tests/views/test_reference_request_review.py

@@ -38,7 +38,7 @@ def test_review_verify(client, seeder, utils, app, mocker, db, db_error, is_veri
     if db_error:
         utils.mock_db_commit(mocker)
 
-    mail_mock = utils.mock_send_mails(mocker)
+    mail_mock = utils.mock_send_mails_async(mocker)
     response = utils.post_form(
         url,
         response,

tests/views/test_verification_request.py

@@ -23,7 +23,7 @@ def test_create(client, app, utils: UtilActions, seeder: Seeder, mocker, db_erro
     if db_error:
         utils.mock_db_commit(mocker)
 
-    mail_mock = utils.mock_send_mails(mocker)
+    mail_mock = utils.mock_send_mails_async(mocker)
     response = utils.post_form(
         url,
         response,

tests/views/test_verification_request_review.py

@@ -51,7 +51,7 @@ def test_review_verify(
     if db_error:
         utils.mock_db_commit(mocker)
 
-    mail_mock = utils.mock_send_mails(mocker)
+    mail_mock = utils.mock_send_mails_async(mocker)
     response = utils.post_form(
         url,
         response,

tests/views/test_widget.py

@@ -197,7 +197,7 @@ def test_event_suggestion_create_for_admin_unit(
     if db_error:
         utils.mock_db_commit(mocker)
 
-    mail_mock = utils.mock_send_mails(mocker)
+    mail_mock = utils.mock_send_mails_async(mocker)
 
     if missing_preview_field:
         del data["accept_tos"]