Performance difference between upload_part and PUT with presigned URL

[harupy]

I'm running the following code to compare the performance between S3.Client.upload_part and PUT with presigned URL (generated by S3.Client.generate_presigned_url):

import cProfile
import os
import tempfile
import uuid
from concurrent.futures import ThreadPoolExecutor
from time import perf_counter

import boto3
import requests

CHUNK_SIZE = 64 * 1024 * 1024


class Timer:
    def __enter__(self):
        self.start = perf_counter()
        return self

    def __exit__(self, type, value, traceback):
        self.time = perf_counter() - self.start


def upload_part(
    s3_client,
    bucket_name,
    file_key,
    upload_id,
    part_number,
    data,
):
    with Timer() as t:
        with cProfile.Profile() as pr:
            res = s3_client.upload_part(
                Bucket=bucket_name,
                Key=file_key,
                UploadId=upload_id,
                PartNumber=part_number,
                Body=data,
                ContentLength=len(data),
            )

            if part_number == 1:
                pr.print_stats()

    return {"PartNumber": part_number, "ETag": res["ETag"]}, t.time


def mpu_upload_part(s3_client, bucket_name, file_key, file_path):
    response = s3_client.create_multipart_upload(Bucket=bucket_name, Key=file_key)
    upload_id = response["UploadId"]

    futures = []
    with ThreadPoolExecutor(10) as executor:
        with open(file_path, "rb") as f:
            part_number = 1
            while data := f.read(CHUNK_SIZE):
                future = executor.submit(
                    upload_part,
                    s3_client=s3_client,
                    bucket_name=bucket_name,
                    file_key=file_key,
                    upload_id=upload_id,
                    part_number=part_number,
                    data=data,
                )
                futures.append(future)
                part_number += 1

    try:
        results = [f.result() for f in futures]
    except:
        s3_client.abort_multipart_upload(Bucket=bucket_name, Key=file_key, UploadId=upload_id)
        raise
    else:
        parts, times = zip(*results)
        average = sum(times) / len(times)
        print(f"Average chunk upload time: {average}")
        s3_client.complete_multipart_upload(
            Bucket=bucket_name,
            Key=file_key,
            UploadId=upload_id,
            MultipartUpload={"Parts": parts},
        )


def upload_presigned_url(
    s3_client,
    bucket_name,
    file_key,
    upload_id,
    part_number,
    data,
    session,
):
    with Timer() as t:
        presigned_url = s3_client.generate_presigned_url(
            "upload_part",
            Params={
                "Bucket": bucket_name,
                "Key": file_key,
                "UploadId": upload_id,
                "PartNumber": part_number,
            },
            ExpiresIn=3600,
        )
        with cProfile.Profile() as pr:
            with session.put(presigned_url, data=data, stream=True) as res:
                res.raise_for_status()

            if part_number == 1:
                pr.print_stats()

    return {"PartNumber": part_number, "ETag": res.headers["ETag"]}, t.time


def mpu_presigned_url(s3_client, bucket_name, file_key, file_path):
    response = s3_client.create_multipart_upload(Bucket=bucket_name, Key=file_key)
    upload_id = response["UploadId"]
    futures = []
    with ThreadPoolExecutor(10) as executor:
        with open(file_path, "rb") as f, requests.Session() as session:
            part_number = 1
            while data := f.read(CHUNK_SIZE):
                future = executor.submit(
                    upload_presigned_url,
                    s3_client=s3_client,
                    bucket_name=bucket_name,
                    file_key=file_key,
                    upload_id=upload_id,
                    part_number=part_number,
                    data=data,
                    session=session,
                )
                futures.append(future)
                part_number += 1

    try:
        results = [f.result() for f in futures]
    except:
        s3_client.abort_multipart_upload(Bucket=bucket_name, Key=file_key, UploadId=upload_id)
        raise
    else:
        parts, times = zip(*results)
        average = sum(times) / len(times)
        print(f"Average chunk upload time: {average}")
        s3_client.complete_multipart_upload(
            Bucket=bucket_name,
            Key=file_key,
            UploadId=upload_id,
            MultipartUpload={"Parts": parts},
        )


def main():
    s3_client = boto3.client(
        "s3",
        aws_access_key_id="xxx",
        aws_secret_access_key="xxx",
    )
    bucket_name = "xxx"  # Replace with your bucket name

    with tempfile.TemporaryDirectory() as t:
        file_path = os.path.join(t, "test")
        with open(file_path, "wb") as f:
            f.seek(5 * 1024 * 1024 * 1024 - 1)
            f.write(b"\0")

        with Timer() as t:
            fname = uuid.uuid4().hex
            mpu_upload_part(s3_client, bucket_name, fname, file_path)
        print(f"upload_part: {t.time:.2f} sec")

        with Timer() as t:
            fname = uuid.uuid4().hex
            mpu_presigned_url(s3_client, bucket_name, fname, file_path)
        print(f"presigned_url: {t.time:.2f} sec")


main()

Result (on i3.8xlarge)

Average chunk upload time: 3.3482334275874677
upload_part: 28.12 sec
Average chunk upload time: 0.6716385396875466
presigned_url: 6.02 sec

I'm curious why upload_part is slower than PUT with presigned URL.

[aBurmeseDev]

HI @harupy - thanks for reaching out and sorry for not getting here sooner. I believe there are many factors to examine when it comes to performance benchmarking. I had some trouble reproducing the same result; I simply performed each method individually and measured the time it took which averaged from 5-9 sec.

I'd be happy to investigate further into it if you could share the boto3 logs, individually for each method if possible.

Here's the code example I used to upload using generate_presigned_url: https://docs.aws.amazon.com/AmazonS3/latest/userguide/example_s3_Scenario_PresignedUrl_section.html