From c3499f1813a5d075652c557733c38f697b008f06 Mon Sep 17 00:00:00 2001
From: Adrian Moennich <adrian.moennich@cern.ch>
Date: Fri, 30 Oct 2020 13:51:17 +0100
Subject: [PATCH] Storage/S3: Use high-level API supporting chunks

Otherwise it's impossible to store files larger than 5GB
---
 storage_s3/indico_storage_s3/storage.py | 15 +++++++++++++--
 storage_s3/setup.py                     |  2 +-
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/storage_s3/indico_storage_s3/storage.py b/storage_s3/indico_storage_s3/storage.py
index a1ae2e3..2b96149 100644
--- a/storage_s3/indico_storage_s3/storage.py
+++ b/storage_s3/indico_storage_s3/storage.py
@@ -18,6 +18,7 @@ from io import BytesIO
 from tempfile import NamedTemporaryFile
 
 import boto3
+from boto3.s3.transfer import TransferConfig
 from botocore.config import Config
 from botocore.exceptions import ClientError
 from werkzeug.datastructures import Headers
@@ -108,8 +109,18 @@ class S3StorageBase(Storage):
         checksum = get_file_checksum(fileobj)
         fileobj.seek(0)
         content_md5 = checksum.decode('hex').encode('base64').strip()
-        self.client.put_object(Body=fileobj, Bucket=bucket, Key=name,
-                               ContentType=content_type, ContentMD5=content_md5)
+        metadata = {
+            'ContentType': content_type,
+            # the md5chksum header is used by tools like rclone in case the etag
+            # is not a md5 (in case of chunked uploads)
+            'Metadata': {'md5chksum': content_md5},
+        }
+        # only switch to chunked upload for large files
+        transfer_config = TransferConfig(
+            multipart_threshold=(512 * 1024 * 1024),
+            multipart_chunksize=(256 * 1024 * 1024)
+        )
+        self.client.upload_fileobj(Fileobj=fileobj, Bucket=bucket, Key=name, ExtraArgs=metadata, Config=transfer_config)
         return checksum
 
     def delete(self, file_id):
diff --git a/storage_s3/setup.py b/storage_s3/setup.py
index 9e9f0e1..c447318 100644
--- a/storage_s3/setup.py
+++ b/storage_s3/setup.py
@@ -12,7 +12,7 @@ from setuptools import find_packages, setup
 
 setup(
     name='indico-plugin-storage-s3',
-    version='2.3.1',
+    version='2.3.2',
     description='S3 storage backend for Indico',
     url='https://github.com/indico/indico-plugins',
     license='MIT',