diff --git a/storage_s3/indico_storage_s3/storage.py b/storage_s3/indico_storage_s3/storage.py index a1ae2e3..2b96149 100644 --- a/storage_s3/indico_storage_s3/storage.py +++ b/storage_s3/indico_storage_s3/storage.py @@ -18,6 +18,7 @@ from io import BytesIO from tempfile import NamedTemporaryFile import boto3 +from boto3.s3.transfer import TransferConfig from botocore.config import Config from botocore.exceptions import ClientError from werkzeug.datastructures import Headers @@ -108,8 +109,18 @@ class S3StorageBase(Storage): checksum = get_file_checksum(fileobj) fileobj.seek(0) content_md5 = checksum.decode('hex').encode('base64').strip() - self.client.put_object(Body=fileobj, Bucket=bucket, Key=name, - ContentType=content_type, ContentMD5=content_md5) + metadata = { + 'ContentType': content_type, + # the md5chksum header is used by tools like rclone in case the etag + # is not a md5 (in case of chunked uploads) + 'Metadata': {'md5chksum': content_md5}, + } + # only switch to chunked upload for large files + transfer_config = TransferConfig( + multipart_threshold=(512 * 1024 * 1024), + multipart_chunksize=(256 * 1024 * 1024) + ) + self.client.upload_fileobj(Fileobj=fileobj, Bucket=bucket, Key=name, ExtraArgs=metadata, Config=transfer_config) return checksum def delete(self, file_id): diff --git a/storage_s3/setup.py b/storage_s3/setup.py index 9e9f0e1..c447318 100644 --- a/storage_s3/setup.py +++ b/storage_s3/setup.py @@ -12,7 +12,7 @@ from setuptools import find_packages, setup setup( name='indico-plugin-storage-s3', - version='2.3.1', + version='2.3.2', description='S3 storage backend for Indico', url='https://github.com/indico/indico-plugins', license='MIT',