diff --git a/benchmark_tests/__init__.py b/benchmark_tests/__init__.py new file mode 100644 index 0000000..0e3cbbd --- /dev/null +++ b/benchmark_tests/__init__.py @@ -0,0 +1 @@ +# Benchmark tests package diff --git a/benchmark_tests/benchmarks.py b/benchmark_tests/benchmarks.py new file mode 100644 index 0000000..8059cb0 --- /dev/null +++ b/benchmark_tests/benchmarks.py @@ -0,0 +1,1417 @@ +from __future__ import annotations + +import json +import os +import random +import sys +import tempfile +from pathlib import Path +from typing import Any + +import boto3 +import pytest +from aiobotocore.session import get_session +from botocore.client import Config + +from conftest import _timeit, _timeit_async_helper +from signurlarity import Client +from signurlarity.aio import AsyncClient + +# ============================================================================= +# OPERATION CONFIGURATION +# ============================================================================= + +BUCKET = "perf-bucket" +KEY = "object.txt" +SRC_KEY = "source.txt" +BODY_1KB = b"x" * 1024 +BODY_SRC = b"source content" +NUM_KEYS = 10 +PREFIX = "bench/" +REGION = "us-east-1" +RNG = random.Random(42) # noqa: S311 + + +# Operation-specific configurations +# Each config defines: iterations, warmup, setup requirements, and call templates +OPERATIONS = [ + "generate_presigned_post", + "generate_presigned_url", + "head_bucket", + "head_object", + "create_bucket", + "delete_objects", + "put_object", + "list_objects", + "copy_object", + "upload_file", +] + + +def _make_key(base: str, rng: random.Random) -> str: + """Generate a unique key with random suffix to avoid memoization.""" + return f"{base}-{rng.randint(0, 1_000_000)}" + + +# ============================================================================= +# SYNC BENCHMARK RUNNERS +# ============================================================================= + + +def _run_generate_presigned_post_sync( + boto_client: boto3.client, + light_client: Client, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark generate_presigned_post for sync clients.""" + iterations = 200 + warmup = 50 + + # Warmup + for _ in range(warmup): + boto_client.generate_presigned_post( + Bucket=BUCKET, Key=KEY, Fields=None, Conditions=None, ExpiresIn=60 + ) + try: + for _ in range(warmup): + light_client.generate_presigned_post( + Bucket=BUCKET, Key=KEY, Fields=None, Conditions=None, ExpiresIn=60 + ) + except NotImplementedError: + if use_cm: + light_client.close() + pytest.skip("signurlarity.Client.generate_presigned_post not implemented") + + def run_boto(n: int): + for _ in range(n): + boto_client.generate_presigned_post( + Bucket=BUCKET, + Key=_make_key(KEY, RNG), + Fields=None, + Conditions=None, + ExpiresIn=60, + ) + + def run_custom(n: int): + for _ in range(n): + light_client.generate_presigned_post( + Bucket=BUCKET, + Key=_make_key(KEY, RNG), + Fields=None, + Conditions=None, + ExpiresIn=60, + ) + + t_boto = _timeit(run_boto, iterations) + t_custom = _timeit(run_custom, iterations) + + if not use_cm: + light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +def _run_generate_presigned_url_sync( + boto_client: boto3.client, + light_client: Client, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark generate_presigned_url for sync clients.""" + iterations = 500 + warmup = 50 + + # Warmup + for _ in range(warmup): + boto_client.generate_presigned_url( + "get_object", Params={"Bucket": BUCKET, "Key": KEY}, ExpiresIn=60 + ) + for _ in range(warmup): + light_client.generate_presigned_url( + "get_object", Params={"Bucket": BUCKET, "Key": KEY}, ExpiresIn=60 + ) + + def run_boto(n: int): + for _ in range(n): + boto_client.generate_presigned_url( + "get_object", + Params={"Bucket": BUCKET, "Key": _make_key(KEY, RNG)}, + ExpiresIn=60, + ) + + def run_custom(n: int): + for _ in range(n): + light_client.generate_presigned_url( + "get_object", + Params={"Bucket": BUCKET, "Key": _make_key(KEY, RNG)}, + ExpiresIn=60, + ) + + t_boto = _timeit(run_boto, iterations) + t_custom = _timeit(run_custom, iterations) + + if not use_cm: + light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +def _run_head_bucket_sync( + boto_client: boto3.client, + light_client: Client, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark head_bucket for sync clients.""" + iterations = 10 + warmup = 10 + + # Setup: create bucket + boto_client.create_bucket(Bucket=BUCKET) + + # Warmup + for _ in range(warmup): + boto_client.head_bucket(Bucket=BUCKET) + for _ in range(warmup): + light_client.head_bucket(Bucket=BUCKET) + + def run_boto(n: int): + for _ in range(n): + boto_client.head_bucket(Bucket=BUCKET) + + def run_custom(n: int): + for _ in range(n): + light_client.head_bucket(Bucket=BUCKET) + + t_boto = _timeit(run_boto, iterations) + t_custom = _timeit(run_custom, iterations) + + if not use_cm: + light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +def _run_head_object_sync( + boto_client: boto3.client, + light_client: Client, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark head_object for sync clients.""" + iterations = 10 + warmup = 10 + + # Setup: create bucket and object + boto_client.create_bucket(Bucket=BUCKET) + boto_client.put_object( + Bucket=BUCKET, Key=KEY, Body=b"test data for head_object perf test" + ) + + # Warmup + for _ in range(warmup): + boto_client.head_object(Bucket=BUCKET, Key=KEY) + for _ in range(warmup): + light_client.head_object(Bucket=BUCKET, Key=KEY) + + def run_boto(n: int): + for _ in range(n): + boto_client.head_object(Bucket=BUCKET, Key=KEY) + + def run_custom(n: int): + for _ in range(n): + light_client.head_object(Bucket=BUCKET, Key=KEY) + + t_boto = _timeit(run_boto, iterations) + t_custom = _timeit(run_custom, iterations) + + if not use_cm: + light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +def _run_create_bucket_sync( + boto_client: boto3.client, + light_client: Client, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark create_bucket for sync clients.""" + iterations = 10 + warmup = 10 + bucket_prefix = "perf-bucket-create" + + # Warmup + for i in range(warmup): + bucket = f"{bucket_prefix}-warmup-{i}" + boto_client.create_bucket(Bucket=bucket) + boto_client.delete_bucket(Bucket=bucket) + + for i in range(warmup): + bucket = f"{bucket_prefix}-warmup-light-{i}" + light_client.create_bucket(Bucket=bucket) + boto_client.delete_bucket(Bucket=bucket) + + def run_boto(n: int): + for i in range(n): + bucket = f"{bucket_prefix}-boto-{i}" + boto_client.create_bucket(Bucket=bucket) + boto_client.delete_bucket(Bucket=bucket) + + def run_custom(n: int): + for i in range(n): + bucket = f"{bucket_prefix}-custom-{i}" + light_client.create_bucket(Bucket=bucket) + boto_client.delete_bucket(Bucket=bucket) + + t_boto = _timeit(run_boto, iterations) + t_custom = _timeit(run_custom, iterations) + + if not use_cm: + light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +def _run_delete_objects_sync( + boto_client: boto3.client, + light_client: Client, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark delete_objects for sync clients.""" + iterations = 10 + warmup = 5 + bucket = "perf-delete-objects" + + # Setup: create bucket + boto_client.create_bucket(Bucket=bucket) + + def _populate(prefix: str): + keys = [f"{prefix}-{i}.txt" for i in range(NUM_KEYS)] + for k in keys: + boto_client.put_object(Bucket=bucket, Key=k, Body=b"data") + return keys + + # Warmup + for i in range(warmup): + keys = _populate(f"warmup-boto-{i}") + boto_client.delete_objects( + Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} + ) + + for i in range(warmup): + keys = _populate(f"warmup-light-{i}") + light_client.delete_objects( + Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} + ) + + def run_boto(n: int): + for i in range(n): + keys = _populate(f"bench-boto-{i}") + boto_client.delete_objects( + Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} + ) + + def run_custom(n: int): + for i in range(n): + keys = _populate(f"bench-light-{i}") + light_client.delete_objects( + Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} + ) + + t_boto = _timeit(run_boto, iterations) + t_custom = _timeit(run_custom, iterations) + + if not use_cm: + light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +def _run_put_object_sync( + boto_client: boto3.client, + light_client: Client, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark put_object for sync clients.""" + iterations = 10 + warmup = 10 + bucket = "perf-put-object" + + # Setup: create bucket + boto_client.create_bucket(Bucket=bucket) + + # Warmup + for i in range(warmup): + boto_client.put_object(Bucket=bucket, Key=f"warmup-boto-{i}.txt", Body=BODY_1KB) + for i in range(warmup): + light_client.put_object( + Bucket=bucket, Key=f"warmup-light-{i}.txt", Body=BODY_1KB + ) + + def run_boto(n: int): + for _ in range(n): + boto_client.put_object( + Bucket=bucket, + Key=f"bench-boto-{RNG.randint(0, 1_000_000)}.txt", + Body=BODY_1KB, + ) + + def run_custom(n: int): + for _ in range(n): + light_client.put_object( + Bucket=bucket, + Key=f"bench-light-{RNG.randint(0, 1_000_000)}.txt", + Body=BODY_1KB, + ) + + t_boto = _timeit(run_boto, iterations) + t_custom = _timeit(run_custom, iterations) + + if not use_cm: + light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +def _run_list_objects_sync( + boto_client: boto3.client, + light_client: Client, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark list_objects for sync clients.""" + iterations = 10 + warmup = 10 + bucket = "perf-list-objects" + + # Setup: create bucket and objects + boto_client.create_bucket(Bucket=bucket) + for i in range(10): + boto_client.put_object(Bucket=bucket, Key=f"{PREFIX}obj-{i}.txt", Body=b"data") + + # Warmup + for _ in range(warmup): + boto_client.list_objects(Bucket=bucket, Prefix=PREFIX) + for _ in range(warmup): + light_client.list_objects(Bucket=bucket, Prefix=PREFIX) + + def run_boto(n: int): + for _ in range(n): + boto_client.list_objects(Bucket=bucket, Prefix=PREFIX) + + def run_custom(n: int): + for _ in range(n): + light_client.list_objects(Bucket=bucket, Prefix=PREFIX) + + t_boto = _timeit(run_boto, iterations) + t_custom = _timeit(run_custom, iterations) + + # Output + print("\n" + "=" * 60) + print("LIST OBJECTS BENCHMARK") + print("=" * 60) + print( + f"boto3 list_objects: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" + ) + print( + f"signurlarity list_objects: {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" + ) + if t_custom > 0: + speedup = t_boto / t_custom + print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") + if speedup > 1: + print(f"✓ Signurlarity implementation is {speedup:.2f}x FASTER!") + else: + print(f"boto3 is {1 / speedup:.2f}x faster") + print("=" * 60) + + if not use_cm: + light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +def _run_copy_object_sync( + boto_client: boto3.client, + light_client: Client, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark copy_object for sync clients.""" + iterations = 10 + warmup = 10 + bucket = "perf-copy-object" + + # Setup: create bucket and source object + boto_client.create_bucket(Bucket=bucket) + boto_client.put_object(Bucket=bucket, Key=SRC_KEY, Body=BODY_SRC) + + # Warmup + for i in range(warmup): + boto_client.copy_object( + Bucket=bucket, + Key=f"warmup-boto-{i}.txt", + CopySource={"Bucket": bucket, "Key": SRC_KEY}, + ) + for i in range(warmup): + light_client.copy_object( + Bucket=bucket, + Key=f"warmup-light-{i}.txt", + CopySource=f"{bucket}/{SRC_KEY}", + ) + + def run_boto(n: int): + for _ in range(n): + boto_client.copy_object( + Bucket=bucket, + Key=f"bench-boto-{RNG.randint(0, 1_000_000)}.txt", + CopySource={"Bucket": bucket, "Key": SRC_KEY}, + ) + + def run_custom(n: int): + for _ in range(n): + light_client.copy_object( + Bucket=bucket, + Key=f"bench-light-{RNG.randint(0, 1_000_000)}.txt", + CopySource=f"{bucket}/{SRC_KEY}", + ) + + t_boto = _timeit(run_boto, iterations) + t_custom = _timeit(run_custom, iterations) + + # Output + print("\n" + "=" * 60) + print("COPY OBJECT BENCHMARK") + print("=" * 60) + print( + f"boto3 copy_object: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" + ) + print( + f"signurlarity copy_object: {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" + ) + if t_custom > 0: + speedup = t_boto / t_custom + print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") + if speedup > 1: + print(f"✓ Signurlarity implementation is {speedup:.2f}x FASTER!") + else: + print(f"boto3 is {1 / speedup:.2f}x faster") + print("=" * 60) + + if not use_cm: + light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +def _run_upload_file_sync( + boto_client: boto3.client, + light_client: Client, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark upload_file for sync clients.""" + iterations = 10 + warmup = 10 + bucket = "perf-upload-file" + + # Setup: create bucket and temp file + boto_client.create_bucket(Bucket=bucket) + + with tempfile.NamedTemporaryFile(delete=False, suffix=".bin") as tmp: + tmp.write(BODY_1KB) + tmp_path = tmp.name + + try: + # Warmup + for i in range(warmup): + boto_client.upload_file( + Filename=tmp_path, Bucket=bucket, Key=f"warmup-boto-{i}.bin" + ) + for i in range(warmup): + light_client.upload_file( + Filename=tmp_path, Bucket=bucket, Key=f"warmup-light-{i}.bin" + ) + + def run_boto(n: int): + for _ in range(n): + boto_client.upload_file( + Filename=tmp_path, + Bucket=bucket, + Key=f"bench-boto-{RNG.randint(0, 1_000_000)}.bin", + ) + + def run_custom(n: int): + for _ in range(n): + light_client.upload_file( + Filename=tmp_path, + Bucket=bucket, + Key=f"bench-light-{RNG.randint(0, 1_000_000)}.bin", + ) + + t_boto = _timeit(run_boto, iterations) + t_custom = _timeit(run_custom, iterations) + finally: + os.unlink(tmp_path) + + # Output + print("\n" + "=" * 60) + print("UPLOAD FILE BENCHMARK") + print("=" * 60) + print( + f"boto3 upload_file: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" + ) + print( + f"signurlarity upload_file: {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" + ) + if t_custom > 0: + speedup = t_boto / t_custom + print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") + if speedup > 1: + print(f"✓ Signurlarity implementation is {speedup:.2f}x FASTER!") + else: + print(f"boto3 is {1 / speedup:.2f}x faster") + print("=" * 60) + + if not use_cm: + light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +# ============================================================================= +# ASYNC BENCHMARK RUNNERS +# ============================================================================= + + +async def _run_generate_presigned_post_async( + boto_client, + async_light_client: AsyncClient, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark generate_presigned_post for async clients.""" + from uuid import uuid4 + + iterations = 500 + warmup = 50 + + # Warmup + for _ in range(warmup): + await boto_client.generate_presigned_post( + Bucket=BUCKET, + Key=KEY + str(uuid4()), + Fields=None, + Conditions=None, + ExpiresIn=60, + ) + try: + for _ in range(warmup): + await async_light_client.generate_presigned_post( + Bucket=BUCKET, + Key=KEY + str(uuid4()), + Fields=None, + Conditions=None, + ExpiresIn=60, + ) + except NotImplementedError: + if use_cm: + await async_light_client.close() + pytest.skip("signurlarity.AsyncClient.generate_presigned_post not implemented") + + async def run_boto(n: int): + for _ in range(n): + await boto_client.generate_presigned_post( + Bucket=BUCKET, + Key=f"{KEY}-{RNG.randint(0, 1_000_000)}", + Fields=None, + Conditions=None, + ExpiresIn=60, + ) + + async def run_custom(n: int): + for _ in range(n): + await async_light_client.generate_presigned_post( + Bucket=BUCKET, + Key=f"{KEY}-{RNG.randint(0, 1_000_000)}", + Fields=None, + Conditions=None, + ExpiresIn=60, + ) + + t_boto = await _timeit_async_helper(run_boto, iterations) + t_custom = await _timeit_async_helper(run_custom, iterations) + + if not use_cm: + await async_light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +async def _run_generate_presigned_url_async( + boto_client, + async_light_client: AsyncClient, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark generate_presigned_url for async clients.""" + iterations = 500 + warmup = 50 + + # Warmup + for _ in range(warmup): + await boto_client.generate_presigned_url( + "get_object", Params={"Bucket": BUCKET, "Key": KEY}, ExpiresIn=60 + ) + for _ in range(warmup): + await async_light_client.generate_presigned_url( + "get_object", Params={"Bucket": BUCKET, "Key": KEY}, ExpiresIn=60 + ) + + async def run_boto(n: int): + for _ in range(n): + await boto_client.generate_presigned_url( + "get_object", + Params={"Bucket": BUCKET, "Key": f"{KEY}-{RNG.randint(0, 1_000_000)}"}, + ExpiresIn=60, + ) + + async def run_custom(n: int): + for _ in range(n): + await async_light_client.generate_presigned_url( + "get_object", + Params={"Bucket": BUCKET, "Key": f"{KEY}-{RNG.randint(0, 1_000_000)}"}, + ExpiresIn=60, + ) + + t_boto = await _timeit_async_helper(run_boto, iterations) + t_custom = await _timeit_async_helper(run_custom, iterations) + + if not use_cm: + await async_light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +async def _run_head_bucket_async( + boto_client, + async_light_client: AsyncClient, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark head_bucket for async clients.""" + iterations = 500 + warmup = 10 + + # Setup: create bucket + await async_light_client.create_bucket(Bucket=BUCKET) + + # Warmup + for _ in range(warmup): + await async_light_client.head_bucket(Bucket=BUCKET) + for _ in range(warmup): + await boto_client.head_bucket(Bucket=BUCKET) + + async def run_boto(n: int): + for _ in range(n): + await boto_client.head_bucket(Bucket=BUCKET) + + async def run_custom(n: int): + for _ in range(n): + await async_light_client.head_bucket(Bucket=BUCKET) + + t_boto = await _timeit_async_helper(run_boto, iterations) + t_custom = await _timeit_async_helper(run_custom, iterations) + + if not use_cm: + await async_light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +async def _run_head_object_async( + boto_client, + async_light_client: AsyncClient, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark head_object for async clients.""" + iterations = 500 + warmup = 10 + key = "perf-object.txt" + + # Setup: create bucket and object + await async_light_client.create_bucket(Bucket=BUCKET) + await boto_client.put_object( + Bucket=BUCKET, Key=key, Body=b"test data for head_object perf test" + ) + + # Warmup + for _ in range(warmup): + await boto_client.head_object(Bucket=BUCKET, Key=key) + for _ in range(warmup): + await async_light_client.head_object(Bucket=BUCKET, Key=key) + + async def run_boto(n: int): + for _ in range(n): + await boto_client.head_object(Bucket=BUCKET, Key=key) + + async def run_custom(n: int): + for _ in range(n): + await async_light_client.head_object(Bucket=BUCKET, Key=key) + + t_boto = await _timeit_async_helper(run_boto, iterations) + t_custom = await _timeit_async_helper(run_custom, iterations) + + # Output + print("\n" + "=" * 60) + print("HEAD OBJECT BENCHMARK (ASYNC)") + print("=" * 60) + print( + f"boto3 head_object: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" + ) + print( + f"signurlarity head_object (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" + ) + if t_custom > 0: + speedup = t_boto / t_custom + print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") + if speedup > 1: + print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") + else: + print(f"boto3 is {1 / speedup:.2f}x faster") + print("=" * 60) + + if not use_cm: + await async_light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +async def _run_create_bucket_async( + boto_client, + async_light_client: AsyncClient, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark create_bucket for async clients.""" + iterations = 500 + warmup = 10 + bucket_prefix = "perf-bucket-create" + + # Warmup + for i in range(warmup): + bucket = f"{bucket_prefix}-warmup-{i}" + await boto_client.create_bucket(Bucket=bucket) + + for i in range(warmup): + bucket = f"{bucket_prefix}-warmup-light-{i}" + await async_light_client.create_bucket(Bucket=bucket) + + async def run_boto(n: int): + for i in range(n): + bucket = f"{bucket_prefix}-boto-{i}" + await boto_client.create_bucket(Bucket=bucket) + + async def run_custom(n: int): + for i in range(n): + bucket = f"{bucket_prefix}-custom-{i}" + await async_light_client.create_bucket(Bucket=bucket) + + t_boto = await _timeit_async_helper(run_boto, iterations) + t_custom = await _timeit_async_helper(run_custom, iterations) + + # Output + print("\n" + "=" * 60) + print("CREATE BUCKET BENCHMARK (ASYNC)") + print("=" * 60) + print( + f"boto3 create_bucket: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" + ) + print( + f"signurlarity create_bucket (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" + ) + if t_custom > 0: + speedup = t_boto / t_custom + print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") + if speedup > 1: + print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") + else: + print(f"boto3 is {1 / speedup:.2f}x faster") + print("=" * 60) + + if not use_cm: + await async_light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +async def _run_delete_objects_async( + boto_client, + async_light_client: AsyncClient, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark delete_objects for async clients.""" + iterations = 10 + warmup = 5 + bucket = "perf-delete-objects" + + # Setup: create bucket + await async_light_client.create_bucket(Bucket=bucket) + + async def _populate(prefix: str): + keys = [f"{prefix}-{i}.txt" for i in range(NUM_KEYS)] + for k in keys: + await boto_client.put_object(Bucket=bucket, Key=k, Body=b"data") + return keys + + # Warmup + for i in range(warmup): + keys = await _populate(f"warmup-boto-{i}") + await boto_client.delete_objects( + Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} + ) + + for i in range(warmup): + keys = await _populate(f"warmup-light-{i}") + await async_light_client.delete_objects( + Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} + ) + + async def run_boto(n: int): + for i in range(n): + keys = await _populate(f"bench-boto-{i}") + await boto_client.delete_objects( + Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} + ) + + async def run_custom(n: int): + for i in range(n): + keys = await _populate(f"bench-light-{i}") + await async_light_client.delete_objects( + Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} + ) + + t_boto = await _timeit_async_helper(run_boto, iterations) + t_custom = await _timeit_async_helper(run_custom, iterations) + + # Output + print("\n" + "=" * 60) + print("DELETE OBJECTS BENCHMARK (ASYNC)") + print("=" * 60) + print( + f"boto3 delete_objects: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" + ) + print( + f"signurlarity delete_objects (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" + ) + if t_custom > 0: + speedup = t_boto / t_custom + print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") + if speedup > 1: + print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") + else: + print(f"boto3 is {1 / speedup:.2f}x faster") + print("=" * 60) + + if not use_cm: + await async_light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +async def _run_put_object_async( + boto_client, + async_light_client: AsyncClient, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark put_object for async clients.""" + iterations = 10 + warmup = 10 + bucket = "perf-put-object-aio" + + # Setup: create bucket + await boto_client.create_bucket(Bucket=bucket) + + # Warmup + for i in range(warmup): + await boto_client.put_object( + Bucket=bucket, Key=f"warmup-boto-{i}.txt", Body=BODY_1KB + ) + for i in range(warmup): + await async_light_client.put_object( + Bucket=bucket, Key=f"warmup-light-{i}.txt", Body=BODY_1KB + ) + + async def run_boto(n: int): + for _ in range(n): + await boto_client.put_object( + Bucket=bucket, + Key=f"bench-boto-{RNG.randint(0, 1_000_000)}.txt", + Body=BODY_1KB, + ) + + async def run_custom(n: int): + for _ in range(n): + await async_light_client.put_object( + Bucket=bucket, + Key=f"bench-light-{RNG.randint(0, 1_000_000)}.txt", + Body=BODY_1KB, + ) + + t_boto = await _timeit_async_helper(run_boto, iterations) + t_custom = await _timeit_async_helper(run_custom, iterations) + + # Output + print("\n" + "=" * 60) + print("PUT OBJECT BENCHMARK (ASYNC)") + print("=" * 60) + print( + f"boto3 put_object: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" + ) + print( + f"signurlarity put_object (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" + ) + if t_custom > 0: + speedup = t_boto / t_custom + print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") + if speedup > 1: + print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") + else: + print(f"boto3 is {1 / speedup:.2f}x faster") + print("=" * 60) + + if not use_cm: + await async_light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +async def _run_list_objects_async( + boto_client, + async_light_client: AsyncClient, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark list_objects for async clients.""" + iterations = 10 + warmup = 10 + bucket = "perf-list-objects-aio" + + # Setup: create bucket and objects + await boto_client.create_bucket(Bucket=bucket) + for i in range(10): + await boto_client.put_object( + Bucket=bucket, Key=f"{PREFIX}obj-{i}.txt", Body=b"data" + ) + + # Warmup + for _ in range(warmup): + await boto_client.list_objects(Bucket=bucket, Prefix=PREFIX) + for _ in range(warmup): + await async_light_client.list_objects(Bucket=bucket, Prefix=PREFIX) + + async def run_boto(n: int): + for _ in range(n): + await boto_client.list_objects(Bucket=bucket, Prefix=PREFIX) + + async def run_custom(n: int): + for _ in range(n): + await async_light_client.list_objects(Bucket=bucket, Prefix=PREFIX) + + t_boto = await _timeit_async_helper(run_boto, iterations) + t_custom = await _timeit_async_helper(run_custom, iterations) + + # Output + print("\n" + "=" * 60) + print("LIST OBJECTS BENCHMARK (ASYNC)") + print("=" * 60) + print( + f"boto3 list_objects: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" + ) + print( + f"signurlarity list_objects (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" + ) + if t_custom > 0: + speedup = t_boto / t_custom + print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") + if speedup > 1: + print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") + else: + print(f"boto3 is {1 / speedup:.2f}x faster") + print("=" * 60) + + if not use_cm: + await async_light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +async def _run_copy_object_async( + boto_client, + async_light_client: AsyncClient, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark copy_object for async clients.""" + iterations = 10 + warmup = 10 + bucket = "perf-copy-object-aio" + + # Setup: create bucket and source object + await boto_client.create_bucket(Bucket=bucket) + await boto_client.put_object(Bucket=bucket, Key=SRC_KEY, Body=BODY_SRC) + + # Warmup + for i in range(warmup): + await boto_client.copy_object( + Bucket=bucket, + Key=f"warmup-boto-{i}.txt", + CopySource={"Bucket": bucket, "Key": SRC_KEY}, + ) + for i in range(warmup): + await async_light_client.copy_object( + Bucket=bucket, + Key=f"warmup-light-{i}.txt", + CopySource=f"{bucket}/{SRC_KEY}", + ) + + async def run_boto(n: int): + for _ in range(n): + await boto_client.copy_object( + Bucket=bucket, + Key=f"bench-boto-{RNG.randint(0, 1_000_000)}.txt", + CopySource={"Bucket": bucket, "Key": SRC_KEY}, + ) + + async def run_custom(n: int): + for _ in range(n): + await async_light_client.copy_object( + Bucket=bucket, + Key=f"bench-light-{RNG.randint(0, 1_000_000)}.txt", + CopySource=f"{bucket}/{SRC_KEY}", + ) + + t_boto = await _timeit_async_helper(run_boto, iterations) + t_custom = await _timeit_async_helper(run_custom, iterations) + + # Output + print("\n" + "=" * 60) + print("COPY OBJECT BENCHMARK (ASYNC)") + print("=" * 60) + print( + f"boto3 copy_object: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" + ) + print( + f"signurlarity copy_object (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" + ) + if t_custom > 0: + speedup = t_boto / t_custom + print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") + if speedup > 1: + print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") + else: + print(f"boto3 is {1 / speedup:.2f}x faster") + print("=" * 60) + + if not use_cm: + await async_light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +async def _run_upload_file_async( + boto_client, + async_light_client: AsyncClient, + test_dir: Path, + use_cm: bool, +) -> dict[str, Any]: + """Benchmark upload_file for async clients.""" + iterations = 10 + warmup = 10 + bucket = "perf-upload-file-aio" + + with tempfile.NamedTemporaryFile(delete=False, suffix=".bin") as tmp: + tmp.write(BODY_1KB) + tmp_path = tmp.name + + try: + # Setup: create bucket + await boto_client.create_bucket(Bucket=bucket) + + # Warmup (aiobotocore has no upload_file; use put_object with file read) + for i in range(warmup): + with open(tmp_path, "rb") as fh: + await boto_client.put_object( + Bucket=bucket, Key=f"warmup-boto-{i}.bin", Body=fh.read() + ) + for i in range(warmup): + await async_light_client.upload_file( + Filename=tmp_path, Bucket=bucket, Key=f"warmup-light-{i}.bin" + ) + + async def run_boto(n: int): + for _ in range(n): + with open(tmp_path, "rb") as fh: + await boto_client.put_object( + Bucket=bucket, + Key=f"bench-boto-{RNG.randint(0, 1_000_000)}.bin", + Body=fh.read(), + ) + + async def run_custom(n: int): + for _ in range(n): + await async_light_client.upload_file( + Filename=tmp_path, + Bucket=bucket, + Key=f"bench-light-{RNG.randint(0, 1_000_000)}.bin", + ) + + t_boto = await _timeit_async_helper(run_boto, iterations) + t_custom = await _timeit_async_helper(run_custom, iterations) + finally: + os.unlink(tmp_path) + + # Output + print("\n" + "=" * 60) + print("UPLOAD FILE BENCHMARK (ASYNC)") + print("=" * 60) + print( + f"aiobotocore put_object (file): {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" + ) + print( + f"signurlarity upload_file (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" + ) + if t_custom > 0: + speedup = t_boto / t_custom + print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") + if speedup > 1: + print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") + else: + print(f"boto3 is {1 / speedup:.2f}x faster") + print("=" * 60) + + if not use_cm: + await async_light_client.close() + + return { + "iterations": iterations, + "boto_total": t_boto, + "signurlarity_total": t_custom, + } + + +# ============================================================================= +# MAIN PARAMETRIZED TEST FUNCTIONS +# ============================================================================= + +# Mapping of operation names to their runner functions +SYNC_RUNNERS = { + "generate_presigned_post": _run_generate_presigned_post_sync, + "generate_presigned_url": _run_generate_presigned_url_sync, + "head_bucket": _run_head_bucket_sync, + "head_object": _run_head_object_sync, + "create_bucket": _run_create_bucket_sync, + "delete_objects": _run_delete_objects_sync, + "put_object": _run_put_object_sync, + "list_objects": _run_list_objects_sync, + "copy_object": _run_copy_object_sync, + "upload_file": _run_upload_file_sync, +} + +ASYNC_RUNNERS = { + "generate_presigned_post": _run_generate_presigned_post_async, + "generate_presigned_url": _run_generate_presigned_url_async, + "head_bucket": _run_head_bucket_async, + "head_object": _run_head_object_async, + "create_bucket": _run_create_bucket_async, + "delete_objects": _run_delete_objects_async, + "put_object": _run_put_object_async, + "list_objects": _run_list_objects_async, + "copy_object": _run_copy_object_async, + "upload_file": _run_upload_file_async, +} + + +# --- Sync benchmarks --- + + +@pytest.mark.parametrize("operation", OPERATIONS) +@pytest.mark.parametrize("use_cm", [False, True]) +def test_benchmark_sync( + operation: str, use_cm: bool, rustfs_server: dict, test_results_dir: Path +): + """Parametrized sync benchmarks: 10 operations × 2 client patterns = 20 tests. + + This replaces test_benchmark.py and test_benchmark_cm.py (1298 lines total). + """ + runner = SYNC_RUNNERS[operation] + py_vers = sys.version_info + + # Create test directory + test_name = f"test_{operation}_perf_sync_{'cm' if use_cm else 'plain'}" + test_dir = test_results_dir / Path(test_name) + os.makedirs(test_dir, exist_ok=True) + result_file = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") + + # Create clients based on pattern + if use_cm: + boto_client = boto3.client("s3", **rustfs_server) + with Client(**rustfs_server) as light_client: + results = runner(boto_client, light_client, test_dir, use_cm=True) + else: + boto_client = boto3.client("s3", **rustfs_server) + light_client = Client(**rustfs_server) + results = runner(boto_client, light_client, test_dir, use_cm=False) + + # Build final results + final_results = { + "python_version": f"{py_vers.major}.{py_vers.minor}", + "tested_method": f"{operation}_sync_{'cm' if use_cm else 'plain'}", + **results, + "boto_ops": results["iterations"] / results["boto_total"], + "signurlarity_ops": results["iterations"] / results["signurlarity_total"], + "speedup": results["boto_total"] / results["signurlarity_total"], + } + + print(final_results) + result_file.write_text(json.dumps(final_results, indent=2)) + + +# --- Async benchmarks --- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", OPERATIONS) +@pytest.mark.parametrize("use_cm", [False, True]) +async def test_benchmark_async( + operation: str, use_cm: bool, rustfs_server: dict, test_results_dir: Path +): + """Parametrized async benchmarks: 10 operations × 2 client patterns = 20 tests. + + This replaces test_benchmark_aio.py and test_benchmark_aio_cm.py (1313 lines total). + """ + runner = ASYNC_RUNNERS[operation] + py_vers = sys.version_info + + # Create test directory + test_name = f"test_{operation}_perf_aio_{'cm' if use_cm else 'plain'}" + test_dir = test_results_dir / Path(test_name) + os.makedirs(test_dir, exist_ok=True) + result_file = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") + + # Create clients based on pattern + session = get_session() + + if use_cm: + async with session.create_client( + "s3", **rustfs_server, config=Config(signature_version="s3v4") + ) as boto_client: + async with AsyncClient(**rustfs_server) as async_light_client: + results = await runner( + boto_client, async_light_client, test_dir, use_cm=True + ) + else: + async with session.create_client( + "s3", **rustfs_server, config=Config(signature_version="s3v4") + ) as boto_client: + async_light_client = AsyncClient(**rustfs_server) + results = await runner( + boto_client, async_light_client, test_dir, use_cm=False + ) + + # Build final results + final_results = { + "python_version": f"{py_vers.major}.{py_vers.minor}", + "tested_method": f"{operation}_aio_{'cm' if use_cm else 'plain'}", + **results, + "boto_ops": results["iterations"] / results["boto_total"], + "signurlarity_ops": results["iterations"] / results["signurlarity_total"], + "speedup": results["boto_total"] / results["signurlarity_total"], + } + + print(final_results) + result_file.write_text(json.dumps(final_results, indent=2)) diff --git a/benchmark_tests/conftest.py b/benchmark_tests/conftest.py index 2667401..7cf630b 100644 --- a/benchmark_tests/conftest.py +++ b/benchmark_tests/conftest.py @@ -1,86 +1,36 @@ -from __future__ import annotations - -import time -from pathlib import Path - -import pytest - - -def pytest_addoption(parser): - parser.addoption( - "--test-results-dir", - type=Path, - default=None, - help="Path to store the perf test results", - ) - - -@pytest.fixture(scope="module") -def test_results_dir(request) -> Path: - test_results_dir = request.config.getoption("--test-results-dir") - if test_results_dir is None: - pytest.skip( - "Requires a directory to store the test results with --test-results-dir" - ) - test_results_dir = test_results_dir.resolve() - yield test_results_dir - +"""Test fixtures for the benchmark_tests directory. -@pytest.fixture(scope="module") -def rustfs_server(): - """Spawn a test rustfs image for benchmarking.""" - AWS_ACCESS_KEY_ID = "rustfsadmin" - AWS_SECRET_ACCESS_KEY = "rustfsadmin" # noqa: S105 - import subprocess - - cmd = [ - "docker", - "run", - "-d", - "--rm", - "--name", - "rustfs_local", - "-p", - "9000:9000", - "-p", - "9001:9001", - "rustfs/rustfs:latest", - "/data", - ] - subprocess.run(cmd, check=True) # noqa: S603 - time.sleep(1) # Wait for server to start - yield { - "endpoint_url": "http://localhost:9000", - "aws_access_key_id": AWS_ACCESS_KEY_ID, - "aws_secret_access_key": AWS_SECRET_ACCESS_KEY, - } - cmd = ["docker", "stop", "rustfs_local"] - subprocess.run(cmd, check=True) # noqa: S603 - - -def _timeit(fn, iterations: int) -> float: - """Measure execution time of a function.""" - import time - - start = time.perf_counter() - fn(iterations) - return time.perf_counter() - start - - -def _timeit_async(fn, iterations: int) -> float: - """Measure execution time of an async function.""" - import asyncio - import time - - start = time.perf_counter() - asyncio.run(fn(iterations)) - return time.perf_counter() - start +This file re-exports constants, utilities, and timing functions from the root conftest.py. +Pytest fixtures defined in root conftest.py are automatically available. +""" +from __future__ import annotations -async def _timeit_async_helper(fn, iterations: int) -> float: - """Measure the execution time of an async function within async context.""" - import time +import importlib.util +import sys +from pathlib import Path - start = time.perf_counter() - await fn(iterations) - return time.perf_counter() - start +# Add project root to path so we can import root_conftest_module +_project_root = str(Path(__file__).parent.parent) +if _project_root not in sys.path: + sys.path.insert(0, _project_root) + +# Import root conftest as a module with a unique name +_root_conftest_path = str(Path(_project_root) / "conftest.py") +spec = importlib.util.spec_from_file_location( + "root_conftest_module", _root_conftest_path +) +root_conftest_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(root_conftest_module) + +# Re-export constants, utilities, and timing functions +# (fixtures are automatically available via pytest, but we need these for direct imports) +BUCKET_NAME = root_conftest_module.BUCKET_NAME +CHECKSUM_ALGORITHM = root_conftest_module.CHECKSUM_ALGORITHM +MISSING_BUCKET_NAME = root_conftest_module.MISSING_BUCKET_NAME +OTHER_BUCKET_NAME = root_conftest_module.OTHER_BUCKET_NAME +_timeit = root_conftest_module._timeit +_timeit_async = root_conftest_module._timeit_async +_timeit_async_helper = root_conftest_module._timeit_async_helper +b16_to_b64 = root_conftest_module.b16_to_b64 +random_file = root_conftest_module.random_file diff --git a/benchmark_tests/test_benchmark.py b/benchmark_tests/test_benchmark.py deleted file mode 100644 index b037bce..0000000 --- a/benchmark_tests/test_benchmark.py +++ /dev/null @@ -1,815 +0,0 @@ -from __future__ import annotations - -import json -import os -import random -import sys -import tempfile -from pathlib import Path - -import boto3 -import pytest -from botocore.client import Config - -from conftest import _timeit -from signurlarity import Client - - -def test_generate_presigned_post_perf_sync(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for presigned POST. - - This is a non-failing, informational test: it prints timings and skips - if the signurlarity implementation is not available. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_generate_presigned_post_perf") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - rng = random.Random(42) # noqa: S311 - bucket = "perf-bucket" - key = "object.txt" - - boto_client = boto3.client("s3", **rustfs_server) - light_client = Client(**rustfs_server) - - # Bucket creation ensures produced URLs are fully valid for the endpoint - # but is not part of the benchmark itself. - # boto_client.create_bucket(Bucket=bucket) - - # Minimal fields/conditions for a fair apples-to-apples comparison - fields = None - conditions = None - - iterations = 200 - - # Warm-up to mitigate one-time costs (imports, JIT-like caches, etc.) - for _ in range(50): - boto_client.generate_presigned_post( - Bucket=bucket, Key=key, Fields=fields, Conditions=conditions, ExpiresIn=60 - ) - try: - for _ in range(50): - light_client.generate_presigned_post( - Bucket=bucket, - Key=key, - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - except NotImplementedError: - pytest.skip( - "signurlarity.Client.generate_presigned_post not implemented; skipping perf comparison" - ) - - def run_boto(n: int): - for _ in range(n): - # Vary key slightly to avoid any internal memoization across loops - boto_client.generate_presigned_post( - Bucket=bucket, - Key=f"{key}-{rng.randint(0, 1_000_000)}", - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - - def run_light(n: int): - for _ in range(n): - light_client.generate_presigned_post( - Bucket=bucket, - Key=f"{key}-{rng.randint(0, 1_000_000)}", - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_light, iterations) - - light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "generate_presigned_post_sync", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print(results) - result_file.write_text(json.dumps(results, indent=2)) - - -def test_generate_presigned_url_perf_sync(rustfs_server, test_results_dir): - """Compare performance of boto3 vs custom S3PresignedURLGenerator for presigned URL. - - This benchmark compares boto3's generate_presigned_url with the custom - implementation that has zero boto3 dependencies. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_generate_presigned_url_perf") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - rng = random.Random(42) # noqa: S311 - bucket = "perf-bucket" - key = "object.txt" - - # Extract region from endpoint_url - region = "us-east-1" - - boto_client = boto3.client( - "s3", - region_name=region, - **rustfs_server, - config=Config(signature_version="s3v4"), - ) - light_client = Client(**rustfs_server) - - # custom_generator = S3PresignedURLGenerator( - # access_key=AWS_ACCESS_KEY_ID, secret_key=AWS_SECRET_ACCESS_KEY, region=region - # ) - - iterations = 500 - - # Warm-up to mitigate one-time costs - for _ in range(50): - boto_client.generate_presigned_url( - "get_object", Params={"Bucket": bucket, "Key": key}, ExpiresIn=60 - ) - - for _ in range(50): - light_client.generate_presigned_url( - "get_object", Params={"Bucket": bucket, "Key": key}, ExpiresIn=60 - ) - - def run_boto(n: int): - for _ in range(n): - # Vary key slightly to avoid any internal memoization - boto_client.generate_presigned_url( - "get_object", - Params={"Bucket": bucket, "Key": f"{key}-{rng.randint(0, 1_000_000)}"}, - ExpiresIn=60, - ) - - def run_custom(n: int): - for _ in range(n): - light_client.generate_presigned_url( - "get_object", - Params={"Bucket": bucket, "Key": f"{key}-{rng.randint(0, 1_000_000)}"}, - ExpiresIn=60, - ) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_custom, iterations) - - light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "generate_presigned_url_sync", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print(results) - result_file.write_text(json.dumps(results, indent=2)) - - -def test_head_bucket_perf_sync(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for head_bucket. - - This benchmark compares boto3's head_bucket with the custom - implementation that has zero boto3 dependencies. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_head_bucket_perf") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - bucket = "perf-bucket" - - # Extract region from endpoint_url - region = "us-east-1" - - boto_client = boto3.client("s3", region_name=region, **rustfs_server) - light_client = Client(**rustfs_server) - - # Create the bucket for testing - boto_client.create_bucket(Bucket=bucket) - - iterations = 10 - - # Warm-up to mitigate one-time costs - for _ in range(10): - boto_client.head_bucket(Bucket=bucket) - - for _ in range(10): - light_client.head_bucket(Bucket=bucket) - - def run_boto(n: int): - for _ in range(n): - boto_client.head_bucket(Bucket=bucket) - - def run_custom(n: int): - for _ in range(n): - light_client.head_bucket(Bucket=bucket) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_custom, iterations) - - light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "head_bucket_sync", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - result_file.write_text(json.dumps(results, indent=2)) - - -def test_head_object_perf_sync(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for head_object. - - This benchmark compares boto3's head_object with the custom - implementation that has zero boto3 dependencies. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_head_object_perf") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - bucket = "perf-object" - key = "perf-object.txt" - - # Extract region from endpoint_url - region = "us-east-1" - - boto_client = boto3.client("s3", region_name=region, **rustfs_server) - light_client = Client(**rustfs_server) - - # Create the bucket and object for testing - boto_client.create_bucket(Bucket=bucket) - boto_client.put_object( - Bucket=bucket, Key=key, Body=b"test data for head_object perf test" - ) - - iterations = 10 - - # Warm-up to mitigate one-time costs - for _ in range(10): - boto_client.head_object(Bucket=bucket, Key=key) - - for _ in range(10): - light_client.head_object(Bucket=bucket, Key=key) - - def run_boto(n: int): - for _ in range(n): - boto_client.head_object(Bucket=bucket, Key=key) - - def run_custom(n: int): - for _ in range(n): - light_client.head_object(Bucket=bucket, Key=key) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_custom, iterations) - - light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "head_object_sync", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - # Informational output - print("\n" + "=" * 60) - print("HEAD OBJECT BENCHMARK") - print("=" * 60) - print( - f"boto3 head_object: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity head_object: {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - - print("=" * 60) - - -def test_create_bucket_perf_sync(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for create_bucket. - - This benchmark compares boto3's create_bucket with the signurlarity - implementation that uses httpx with AWS Signature V4. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_create_bucket_perf") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - boto_client = boto3.client("s3", **rustfs_server) - light_client = Client(**rustfs_server) - - iterations = 10 - bucket_prefix = "perf-bucket-create" - - # Warm-up to mitigate one-time costs - for i in range(10): - bucket = f"{bucket_prefix}-warmup-{i}" - boto_client.create_bucket(Bucket=bucket) - boto_client.delete_bucket(Bucket=bucket) - - for i in range(10): - bucket = f"{bucket_prefix}-warmup-light-{i}" - light_client.create_bucket(Bucket=bucket) - boto_client.delete_bucket(Bucket=bucket) - - def run_boto(n: int): - for i in range(n): - bucket = f"{bucket_prefix}-boto-{i}" - boto_client.create_bucket(Bucket=bucket) - boto_client.delete_bucket(Bucket=bucket) - - def run_custom(n: int): - for i in range(n): - bucket = f"{bucket_prefix}-custom-{i}" - light_client.create_bucket(Bucket=bucket) - boto_client.delete_bucket(Bucket=bucket) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_custom, iterations) - - light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "create_bucket_sync", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - # Informational output - print("\n" + "=" * 60) - print("CREATE BUCKET BENCHMARK") - print("=" * 60) - print( - f"boto3 create_bucket: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity create_bucket: {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - - print("=" * 60) - - -def test_delete_objects_perf_sync(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for delete_objects. - - This benchmark compares boto3's delete_objects with the signurlarity - implementation that uses httpx with AWS Signature V4. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_delete_objects_perf") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - bucket = "perf-delete-objects" - num_keys = 10 - - boto_client = boto3.client("s3", **rustfs_server) - light_client = Client(**rustfs_server) - - # Create the bucket for testing - boto_client.create_bucket(Bucket=bucket) - - iterations = 10 - - def _populate(prefix: str): - keys = [f"{prefix}-{i}.txt" for i in range(num_keys)] - for k in keys: - boto_client.put_object(Bucket=bucket, Key=k, Body=b"data") - return keys - - # Warm-up - for i in range(5): - keys = _populate(f"warmup-boto-{i}") - boto_client.delete_objects( - Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} - ) - - for i in range(5): - keys = _populate(f"warmup-light-{i}") - light_client.delete_objects( - Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} - ) - - def run_boto(n: int): - for i in range(n): - keys = _populate(f"bench-boto-{i}") - boto_client.delete_objects( - Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} - ) - - def run_custom(n: int): - for i in range(n): - keys = _populate(f"bench-light-{i}") - light_client.delete_objects( - Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} - ) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_custom, iterations) - - light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "delete_objects_sync", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print("\n" + "=" * 60) - print("DELETE OBJECTS BENCHMARK") - print("=" * 60) - print( - f"boto3 delete_objects: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity delete_objects: {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - - print("=" * 60) - - -def test_put_object_perf_sync(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for put_object. - - Uploads a 1 KB object per iteration to a unique key, comparing boto3 and - signurlarity implementations. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_put_object_perf") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - rng = random.Random(42) # noqa: S311 - bucket = "perf-put-object" - body = b"x" * 1024 # 1 KB - - boto_client = boto3.client("s3", **rustfs_server) - light_client = Client(**rustfs_server) - - boto_client.create_bucket(Bucket=bucket) - - iterations = 10 - - # Warm-up - for i in range(10): - boto_client.put_object(Bucket=bucket, Key=f"warmup-boto-{i}.txt", Body=body) - for i in range(10): - light_client.put_object(Bucket=bucket, Key=f"warmup-light-{i}.txt", Body=body) - - def run_boto(n: int): - for _ in range(n): - boto_client.put_object( - Bucket=bucket, - Key=f"bench-boto-{rng.randint(0, 1_000_000)}.txt", - Body=body, - ) - - def run_custom(n: int): - for _ in range(n): - light_client.put_object( - Bucket=bucket, - Key=f"bench-light-{rng.randint(0, 1_000_000)}.txt", - Body=body, - ) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_custom, iterations) - - light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "put_object_sync", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print("\n" + "=" * 60) - print("PUT OBJECT BENCHMARK") - print("=" * 60) - print( - f"boto3 put_object: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity put_object: {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - print("=" * 60) - - -def test_list_objects_perf_sync(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for list_objects. - - Pre-populates 10 objects and benchmarks listing them with a prefix filter. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_list_objects_perf") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - bucket = "perf-list-objects" - prefix = "bench/" - - boto_client = boto3.client("s3", **rustfs_server) - light_client = Client(**rustfs_server) - - boto_client.create_bucket(Bucket=bucket) - for i in range(10): - boto_client.put_object(Bucket=bucket, Key=f"{prefix}obj-{i}.txt", Body=b"data") - - iterations = 10 - - # Warm-up - for _ in range(10): - boto_client.list_objects(Bucket=bucket, Prefix=prefix) - for _ in range(10): - light_client.list_objects(Bucket=bucket, Prefix=prefix) - - def run_boto(n: int): - for _ in range(n): - boto_client.list_objects(Bucket=bucket, Prefix=prefix) - - def run_custom(n: int): - for _ in range(n): - light_client.list_objects(Bucket=bucket, Prefix=prefix) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_custom, iterations) - - light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "list_objects_sync", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print("\n" + "=" * 60) - print("LIST OBJECTS BENCHMARK") - print("=" * 60) - print( - f"boto3 list_objects: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity list_objects: {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - print("=" * 60) - - -def test_copy_object_perf_sync(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for copy_object. - - Pre-uploads a source object and benchmarks copying it to unique destination keys. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_copy_object_perf") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - rng = random.Random(42) # noqa: S311 - bucket = "perf-copy-object" - src_key = "source.txt" - - boto_client = boto3.client("s3", **rustfs_server) - light_client = Client(**rustfs_server) - - boto_client.create_bucket(Bucket=bucket) - boto_client.put_object(Bucket=bucket, Key=src_key, Body=b"source content") - - iterations = 10 - - # Warm-up - for i in range(10): - boto_client.copy_object( - Bucket=bucket, - Key=f"warmup-boto-{i}.txt", - CopySource={"Bucket": bucket, "Key": src_key}, - ) - for i in range(10): - light_client.copy_object( - Bucket=bucket, - Key=f"warmup-light-{i}.txt", - CopySource=f"{bucket}/{src_key}", - ) - - def run_boto(n: int): - for _ in range(n): - boto_client.copy_object( - Bucket=bucket, - Key=f"bench-boto-{rng.randint(0, 1_000_000)}.txt", - CopySource={"Bucket": bucket, "Key": src_key}, - ) - - def run_custom(n: int): - for _ in range(n): - light_client.copy_object( - Bucket=bucket, - Key=f"bench-light-{rng.randint(0, 1_000_000)}.txt", - CopySource=f"{bucket}/{src_key}", - ) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_custom, iterations) - - light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "copy_object_sync", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print("\n" + "=" * 60) - print("COPY OBJECT BENCHMARK") - print("=" * 60) - print( - f"boto3 copy_object: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity copy_object: {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - print("=" * 60) - - -def test_upload_file_perf_sync(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for upload_file. - - Uses a 1 KB temporary file and uploads to unique keys per iteration. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_upload_file_perf") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - rng = random.Random(42) # noqa: S311 - bucket = "perf-upload-file" - - boto_client = boto3.client("s3", **rustfs_server) - light_client = Client(**rustfs_server) - - boto_client.create_bucket(Bucket=bucket) - - iterations = 10 - - with tempfile.NamedTemporaryFile(delete=False, suffix=".bin") as tmp: - tmp.write(b"y" * 1024) # 1 KB - tmp_path = tmp.name - - try: - # Warm-up - for i in range(10): - boto_client.upload_file( - Filename=tmp_path, Bucket=bucket, Key=f"warmup-boto-{i}.bin" - ) - for i in range(10): - light_client.upload_file( - Filename=tmp_path, Bucket=bucket, Key=f"warmup-light-{i}.bin" - ) - - def run_boto(n: int): - for _ in range(n): - boto_client.upload_file( - Filename=tmp_path, - Bucket=bucket, - Key=f"bench-boto-{rng.randint(0, 1_000_000)}.bin", - ) - - def run_custom(n: int): - for _ in range(n): - light_client.upload_file( - Filename=tmp_path, - Bucket=bucket, - Key=f"bench-light-{rng.randint(0, 1_000_000)}.bin", - ) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_custom, iterations) - finally: - os.unlink(tmp_path) - light_client.close() - - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "upload_file_sync", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print("\n" + "=" * 60) - print("UPLOAD FILE BENCHMARK") - print("=" * 60) - print( - f"boto3 upload_file: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity upload_file: {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - print("=" * 60) diff --git a/benchmark_tests/test_benchmark_aio.py b/benchmark_tests/test_benchmark_aio.py deleted file mode 100644 index d19eaf8..0000000 --- a/benchmark_tests/test_benchmark_aio.py +++ /dev/null @@ -1,834 +0,0 @@ -from __future__ import annotations - -import json -import os -import random -import sys -import tempfile -from pathlib import Path -from uuid import uuid4 - -import pytest -from aiobotocore.session import get_session -from botocore.client import Config - -from conftest import _timeit_async_helper -from signurlarity.aio import AsyncClient - - -@pytest.mark.asyncio -async def test_generate_presigned_post_perf_aio(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for presigned POST (async). - - This is a non-failing, informational test: it prints timings and skips - if the signurlarity implementation is not available. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_generate_presigned_post_perf_aio") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - rng = random.Random(42) # noqa: S311 - bucket = "perf-bucket" - key = "object.txt" - - session = get_session() - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - async_light_client = AsyncClient( - **rustfs_server, - ) - - # Minimal fields/conditions for a fair apples-to-apples comparison - fields = None - conditions = None - - iterations = 500 - - # Warm-up to mitigate one-time costs (imports, JIT-like caches, etc.) - for _ in range(50): - await boto_client.generate_presigned_post( - Bucket=bucket, - Key=key + str(uuid4()), - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - for _ in range(50): - await async_light_client.generate_presigned_post( - Bucket=bucket, - Key=key + str(uuid4()), - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - - async def run_boto(n: int): - for _ in range(n): - await boto_client.generate_presigned_post( - Bucket=bucket, - Key=f"{key}-{rng.randint(0, 1_000_000)}", - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - - async def run_light(n: int): - for _ in range(n): - await async_light_client.generate_presigned_post( - Bucket=bucket, - Key=f"{key}-{rng.randint(0, 1_000_000)}", - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - - t_boto = await _timeit_async_helper(run_boto, iterations) - t_custom = await _timeit_async_helper(run_light, iterations) - - await async_light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "generate_presigned_post_aio", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print(results) - result_file.write_text(json.dumps(results, indent=2)) - - -@pytest.mark.asyncio -async def test_generate_presigned_url_perf_aio(rustfs_server, test_results_dir): - """Compare performance of signurlarity async for presigned URL (async). - - This benchmark tests the async implementation's presigned URL generation. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_generate_presigned_url_perf_aio") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - rng = random.Random(42) # noqa: S311 - bucket = "perf-bucket" - key = "object.txt" - - async_light_client = AsyncClient(**rustfs_server) - session = get_session() - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - iterations = 500 - - # Warm-up to mitigate one-time costs - for _ in range(50): - await boto_client.generate_presigned_url( - "get_object", Params={"Bucket": bucket, "Key": key}, ExpiresIn=60 - ) - - for _ in range(50): - await async_light_client.generate_presigned_url( - "get_object", Params={"Bucket": bucket, "Key": key}, ExpiresIn=60 - ) - - async def run_boto(n: int): - for _ in range(n): - await boto_client.generate_presigned_url( - "get_object", - Params={ - "Bucket": bucket, - "Key": f"{key}-{rng.randint(0, 1_000_000)}", - }, - ExpiresIn=60, - ) - - async def run_custom(n: int): - for _ in range(n): - await async_light_client.generate_presigned_url( - "get_object", - Params={ - "Bucket": bucket, - "Key": f"{key}-{rng.randint(0, 1_000_000)}", - }, - ExpiresIn=60, - ) - - t_boto = await _timeit_async_helper(run_boto, iterations) - t_custom = await _timeit_async_helper(run_custom, iterations) - - await async_light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "generate_presigned_url_aio", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print(results) - result_file.write_text(json.dumps(results, indent=2)) - - -@pytest.mark.asyncio -async def test_head_bucket_perf_aio(rustfs_server, test_results_dir): - """Compare performance of signurlarity async for head_bucket. - - This benchmark tests the async implementation's head_bucket functionality. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_head_bucket_perf_aio") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - bucket = "perf-bucket" - - async_light_client = AsyncClient(**rustfs_server) - session = get_session() - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - # Create the bucket for testing - await async_light_client.create_bucket(Bucket=bucket) - - iterations = 500 - - # Warm-up to mitigate one-time costs - for _ in range(10): - await async_light_client.head_bucket(Bucket=bucket) - - # Warm-up to mitigate one-time costs - for _ in range(10): - await boto_client.head_bucket(Bucket=bucket) - - async def run_boto(n: int): - for _ in range(n): - await boto_client.head_bucket(Bucket=bucket) - - async def run_custom(n: int): - for _ in range(n): - await async_light_client.head_bucket(Bucket=bucket) - - t_boto = await _timeit_async_helper(run_custom, iterations) - t_custom = await _timeit_async_helper(run_custom, iterations) - - await async_light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "head_bucket_aio", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - result_file.write_text(json.dumps(results, indent=2)) - - -@pytest.mark.asyncio -async def test_head_object_perf_aio(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity async for head_object. - - This benchmark tests the async implementation's head_object functionality. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_head_object_perf_aio") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - bucket = "perf-object" - key = "perf-object.txt" - - async_light_client = AsyncClient(**rustfs_server) - session = get_session() - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - # Create the bucket and object for testing - await async_light_client.create_bucket(Bucket=bucket) - await boto_client.put_object( - Bucket=bucket, Key=key, Body=b"test data for head_object perf test" - ) - - iterations = 500 - # Warm-up to mitigate one-time costs - for _ in range(10): - await boto_client.head_object(Bucket=bucket, Key=key) - - for _ in range(10): - await async_light_client.head_object(Bucket=bucket, Key=key) - - async def run_boto(n: int): - for _ in range(n): - await boto_client.head_object(Bucket=bucket, Key=key) - - async def run_custom(n: int): - for _ in range(n): - await async_light_client.head_object(Bucket=bucket, Key=key) - - t_boto = await _timeit_async_helper(run_boto, iterations) - t_custom = await _timeit_async_helper(run_custom, iterations) - - await async_light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "head_object_aio", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - # Informational output - print("\n" + "=" * 60) - print("HEAD OBJECT BENCHMARK (ASYNC)") - print("=" * 60) - print( - f"boto3 head_object: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity head_object (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - - print("=" * 60) - - -@pytest.mark.asyncio -async def test_create_bucket_perf_aio(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity async for create_bucket. - - This benchmark tests the async implementation's create_bucket functionality. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_create_bucket_perf_aio") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - async_light_client = AsyncClient(**rustfs_server) - session = get_session() - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - iterations = 500 - bucket_prefix = "perf-bucket-create" - - # Warm-up to mitigate one-time costs - for i in range(10): - bucket = f"{bucket_prefix}-warmup-{i}" - - await boto_client.create_bucket(Bucket=bucket) - - for i in range(10): - bucket = f"{bucket_prefix}-warmup-light-{i}" - - await async_light_client.create_bucket(Bucket=bucket) - - async def run_boto(n: int): - for i in range(n): - bucket = f"{bucket_prefix}-boto-{i}" - - await boto_client.create_bucket(Bucket=bucket) - - async def run_custom(n: int): - for i in range(n): - bucket = f"{bucket_prefix}-custom-{i}" - - await async_light_client.create_bucket(Bucket=bucket) - - t_boto = await _timeit_async_helper(run_boto, iterations) - t_custom = await _timeit_async_helper(run_custom, iterations) - - await async_light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "create_bucket_aio", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - # Informational output - print("\n" + "=" * 60) - print("CREATE BUCKET BENCHMARK (ASYNC)") - print("=" * 60) - print( - f"boto3 create_bucket: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity create_bucket (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - - print("=" * 60) - - -@pytest.mark.asyncio -async def test_delete_objects_perf_aio(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity async for delete_objects. - - This benchmark tests the async implementation's delete_objects functionality. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_delete_objects_perf_aio") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - bucket = "perf-delete-objects" - num_keys = 10 - - async_light_client = AsyncClient(**rustfs_server) - session = get_session() - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - # Create the bucket for testing - await async_light_client.create_bucket(Bucket=bucket) - - iterations = 10 - - async def _populate(prefix: str): - keys = [f"{prefix}-{i}.txt" for i in range(num_keys)] - for k in keys: - await boto_client.put_object(Bucket=bucket, Key=k, Body=b"data") - return keys - - # Warm-up - for i in range(5): - keys = await _populate(f"warmup-boto-{i}") - await boto_client.delete_objects( - Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} - ) - - for i in range(5): - keys = await _populate(f"warmup-light-{i}") - await async_light_client.delete_objects( - Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} - ) - - async def run_boto(n: int): - for i in range(n): - keys = await _populate(f"bench-boto-{i}") - await boto_client.delete_objects( - Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} - ) - - async def run_custom(n: int): - for i in range(n): - keys = await _populate(f"bench-light-{i}") - await async_light_client.delete_objects( - Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} - ) - - t_boto = await _timeit_async_helper(run_boto, iterations) - t_custom = await _timeit_async_helper(run_custom, iterations) - - await async_light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "delete_objects_aio", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print("\n" + "=" * 60) - print("DELETE OBJECTS BENCHMARK (ASYNC)") - print("=" * 60) - print( - f"boto3 delete_objects: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity delete_objects (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - - print("=" * 60) - - -@pytest.mark.asyncio -async def test_put_object_perf_aio(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity async for put_object. - - Uploads a 1 KB object per iteration to a unique key. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_put_object_perf_aio") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - rng = random.Random(42) # noqa: S311 - bucket = "perf-put-object-aio" - body = b"x" * 1024 # 1 KB - - async_light_client = AsyncClient(**rustfs_server) - session = get_session() - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - await boto_client.create_bucket(Bucket=bucket) - - iterations = 10 - - # Warm-up - for i in range(10): - await boto_client.put_object( - Bucket=bucket, Key=f"warmup-boto-{i}.txt", Body=body - ) - for i in range(10): - await async_light_client.put_object( - Bucket=bucket, Key=f"warmup-light-{i}.txt", Body=body - ) - - async def run_boto(n: int): - for _ in range(n): - await boto_client.put_object( - Bucket=bucket, - Key=f"bench-boto-{rng.randint(0, 1_000_000)}.txt", - Body=body, - ) - - async def run_custom(n: int): - for _ in range(n): - await async_light_client.put_object( - Bucket=bucket, - Key=f"bench-light-{rng.randint(0, 1_000_000)}.txt", - Body=body, - ) - - t_boto = await _timeit_async_helper(run_boto, iterations) - t_custom = await _timeit_async_helper(run_custom, iterations) - - await async_light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "put_object_aio", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print("\n" + "=" * 60) - print("PUT OBJECT BENCHMARK (ASYNC)") - print("=" * 60) - print( - f"boto3 put_object: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity put_object (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - print("=" * 60) - - -@pytest.mark.asyncio -async def test_list_objects_perf_aio(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity async for list_objects. - - Pre-populates 10 objects and benchmarks listing them with a prefix filter. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_list_objects_perf_aio") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - bucket = "perf-list-objects-aio" - prefix = "bench/" - - async_light_client = AsyncClient(**rustfs_server) - session = get_session() - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - await boto_client.create_bucket(Bucket=bucket) - for i in range(10): - await boto_client.put_object( - Bucket=bucket, Key=f"{prefix}obj-{i}.txt", Body=b"data" - ) - - iterations = 10 - - # Warm-up - for _ in range(10): - await boto_client.list_objects(Bucket=bucket, Prefix=prefix) - for _ in range(10): - await async_light_client.list_objects(Bucket=bucket, Prefix=prefix) - - async def run_boto(n: int): - for _ in range(n): - await boto_client.list_objects(Bucket=bucket, Prefix=prefix) - - async def run_custom(n: int): - for _ in range(n): - await async_light_client.list_objects(Bucket=bucket, Prefix=prefix) - - t_boto = await _timeit_async_helper(run_boto, iterations) - t_custom = await _timeit_async_helper(run_custom, iterations) - - await async_light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "list_objects_aio", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print("\n" + "=" * 60) - print("LIST OBJECTS BENCHMARK (ASYNC)") - print("=" * 60) - print( - f"boto3 list_objects: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity list_objects (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - print("=" * 60) - - -@pytest.mark.asyncio -async def test_copy_object_perf_aio(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity async for copy_object. - - Pre-uploads a source object and benchmarks copying it to unique destination keys. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_copy_object_perf_aio") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - rng = random.Random(42) # noqa: S311 - bucket = "perf-copy-object-aio" - src_key = "source.txt" - - async_light_client = AsyncClient(**rustfs_server) - session = get_session() - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - await boto_client.create_bucket(Bucket=bucket) - await boto_client.put_object(Bucket=bucket, Key=src_key, Body=b"source content") - - iterations = 10 - - # Warm-up - for i in range(10): - await boto_client.copy_object( - Bucket=bucket, - Key=f"warmup-boto-{i}.txt", - CopySource={"Bucket": bucket, "Key": src_key}, - ) - for i in range(10): - await async_light_client.copy_object( - Bucket=bucket, - Key=f"warmup-light-{i}.txt", - CopySource=f"{bucket}/{src_key}", - ) - - async def run_boto(n: int): - for _ in range(n): - await boto_client.copy_object( - Bucket=bucket, - Key=f"bench-boto-{rng.randint(0, 1_000_000)}.txt", - CopySource={"Bucket": bucket, "Key": src_key}, - ) - - async def run_custom(n: int): - for _ in range(n): - await async_light_client.copy_object( - Bucket=bucket, - Key=f"bench-light-{rng.randint(0, 1_000_000)}.txt", - CopySource=f"{bucket}/{src_key}", - ) - - t_boto = await _timeit_async_helper(run_boto, iterations) - t_custom = await _timeit_async_helper(run_custom, iterations) - - await async_light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "copy_object_aio", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print("\n" + "=" * 60) - print("COPY OBJECT BENCHMARK (ASYNC)") - print("=" * 60) - print( - f"boto3 copy_object: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity copy_object (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - print("=" * 60) - - -@pytest.mark.asyncio -async def test_upload_file_perf_aio(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity async for upload_file. - - Uses a 1 KB temporary file and uploads to unique keys per iteration. - aiobotocore has no upload_file; boto reference uses put_object with file read. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_upload_file_perf_aio") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - rng = random.Random(42) # noqa: S311 - bucket = "perf-upload-file-aio" - - async_light_client = AsyncClient(**rustfs_server) - session = get_session() - - with tempfile.NamedTemporaryFile(delete=False, suffix=".bin") as tmp: - tmp.write(b"y" * 1024) # 1 KB - tmp_path = tmp.name - - try: - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - await boto_client.create_bucket(Bucket=bucket) - - iterations = 10 - - # Warm-up (aiobotocore has no upload_file; use put_object with file read) - for i in range(10): - with open(tmp_path, "rb") as fh: # noqa: PTH123 - await boto_client.put_object( - Bucket=bucket, Key=f"warmup-boto-{i}.bin", Body=fh.read() - ) - for i in range(10): - await async_light_client.upload_file( - Filename=tmp_path, Bucket=bucket, Key=f"warmup-light-{i}.bin" - ) - - async def run_boto(n: int): - for _ in range(n): - with open(tmp_path, "rb") as fh: # noqa: PTH123 - await boto_client.put_object( - Bucket=bucket, - Key=f"bench-boto-{rng.randint(0, 1_000_000)}.bin", - Body=fh.read(), - ) - - async def run_custom(n: int): - for _ in range(n): - await async_light_client.upload_file( - Filename=tmp_path, - Bucket=bucket, - Key=f"bench-light-{rng.randint(0, 1_000_000)}.bin", - ) - - t_boto = await _timeit_async_helper(run_boto, iterations) - t_custom = await _timeit_async_helper(run_custom, iterations) - finally: - os.unlink(tmp_path) - - await async_light_client.close() - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "upload_file_aio", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print("\n" + "=" * 60) - print("UPLOAD FILE BENCHMARK (ASYNC)") - print("=" * 60) - print( - f"aiobotocore put_object (file): {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity upload_file (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - print("=" * 60) diff --git a/benchmark_tests/test_benchmark_aio_cm.py b/benchmark_tests/test_benchmark_aio_cm.py deleted file mode 100644 index 732bf03..0000000 --- a/benchmark_tests/test_benchmark_aio_cm.py +++ /dev/null @@ -1,479 +0,0 @@ -from __future__ import annotations - -import json -import os -import random -import sys -from pathlib import Path -from uuid import uuid4 - -import pytest -from aiobotocore.session import get_session -from botocore.client import Config - -from conftest import _timeit_async_helper -from signurlarity.aio import AsyncClient - - -@pytest.mark.asyncio -async def test_generate_presigned_post_perf_aio_cm(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for presigned POST (async). - - This is a non-failing, informational test: it prints timings and skips - if the signurlarity implementation is not available. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_generate_presigned_post_perf_aio_cm") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - rng = random.Random(42) # noqa: S311 - bucket = "perf-bucket" - key = "object.txt" - - session = get_session() - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - async with AsyncClient( - **rustfs_server, - ) as async_light_client: - # Minimal fields/conditions for a fair apples-to-apples comparison - fields = None - conditions = None - - iterations = 500 - - # Warm-up to mitigate one-time costs (imports, JIT-like caches, etc.) - for _ in range(50): - await boto_client.generate_presigned_post( - Bucket=bucket, - Key=key + str(uuid4()), - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - for _ in range(50): - await async_light_client.generate_presigned_post( - Bucket=bucket, - Key=key + str(uuid4()), - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - - async def run_boto(n: int): - for _ in range(n): - await boto_client.generate_presigned_post( - Bucket=bucket, - Key=f"{key}-{rng.randint(0, 1_000_000)}", - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - - async def run_light(n: int): - for _ in range(n): - await async_light_client.generate_presigned_post( - Bucket=bucket, - Key=f"{key}-{rng.randint(0, 1_000_000)}", - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - - t_boto = await _timeit_async_helper(run_boto, iterations) - t_custom = await _timeit_async_helper(run_light, iterations) - - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "generate_presigned_post_aio_cm", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print(results) - result_file.write_text(json.dumps(results, indent=2)) - - -@pytest.mark.asyncio -async def test_generate_presigned_url_perf_aio_cm(rustfs_server, test_results_dir): - """Compare performance of signurlarity async for presigned URL (async). - - This benchmark tests the async implementation's presigned URL generation. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_generate_presigned_url_perf_aio_cm") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - rng = random.Random(42) # noqa: S311 - bucket = "perf-bucket" - key = "object.txt" - - session = get_session() - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - async with AsyncClient(**rustfs_server) as async_light_client: - iterations = 500 - - # Warm-up to mitigate one-time costs - for _ in range(50): - await boto_client.generate_presigned_url( - "get_object", Params={"Bucket": bucket, "Key": key}, ExpiresIn=60 - ) - - for _ in range(50): - await async_light_client.generate_presigned_url( - "get_object", Params={"Bucket": bucket, "Key": key}, ExpiresIn=60 - ) - - async def run_boto(n: int): - for _ in range(n): - await boto_client.generate_presigned_url( - "get_object", - Params={ - "Bucket": bucket, - "Key": f"{key}-{rng.randint(0, 1_000_000)}", - }, - ExpiresIn=60, - ) - - async def run_custom(n: int): - for _ in range(n): - await async_light_client.generate_presigned_url( - "get_object", - Params={ - "Bucket": bucket, - "Key": f"{key}-{rng.randint(0, 1_000_000)}", - }, - ExpiresIn=60, - ) - - t_boto = await _timeit_async_helper(run_boto, iterations) - t_custom = await _timeit_async_helper(run_custom, iterations) - - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "generate_presigned_url_aio_cm", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print(results) - result_file.write_text(json.dumps(results, indent=2)) - - -@pytest.mark.asyncio -async def test_head_bucket_perf_aio_cm(rustfs_server, test_results_dir): - """Compare performance of signurlarity async for head_bucket. - - This benchmark tests the async implementation's head_bucket functionality. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_head_bucket_perf_aio_cm") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - bucket = "perf-bucket" - - session = get_session() - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - async with AsyncClient(**rustfs_server) as async_light_client: - # Create the bucket for testing - await async_light_client.create_bucket(Bucket=bucket) - - iterations = 500 - - # Warm-up to mitigate one-time costs - for _ in range(10): - await async_light_client.head_bucket(Bucket=bucket) - - # Warm-up to mitigate one-time costs - for _ in range(10): - await boto_client.head_bucket(Bucket=bucket) - - async def run_boto(n: int): - for _ in range(n): - await boto_client.head_bucket(Bucket=bucket) - - async def run_custom(n: int): - for _ in range(n): - await async_light_client.head_bucket(Bucket=bucket) - - t_boto = await _timeit_async_helper(run_custom, iterations) - t_custom = await _timeit_async_helper(run_custom, iterations) - - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "head_bucket_aio_cm", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - result_file.write_text(json.dumps(results, indent=2)) - - -@pytest.mark.asyncio -async def test_head_object_perf_aio_cm(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity async for head_object. - - This benchmark tests the async implementation's head_object functionality. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_head_object_perf_aio_cm") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - bucket = "perf-object" - key = "perf-object.txt" - - session = get_session() - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - async with AsyncClient(**rustfs_server) as async_light_client: - # Create the bucket and object for testing - await async_light_client.create_bucket(Bucket=bucket) - await boto_client.put_object( - Bucket=bucket, Key=key, Body=b"test data for head_object perf test" - ) - - iterations = 500 - # Warm-up to mitigate one-time costs - for _ in range(10): - await boto_client.head_object(Bucket=bucket, Key=key) - - for _ in range(10): - await async_light_client.head_object(Bucket=bucket, Key=key) - - async def run_boto(n: int): - for _ in range(n): - await boto_client.head_object(Bucket=bucket, Key=key) - - async def run_custom(n: int): - for _ in range(n): - await async_light_client.head_object(Bucket=bucket, Key=key) - - t_boto = await _timeit_async_helper(run_boto, iterations) - t_custom = await _timeit_async_helper(run_custom, iterations) - - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "head_object_aio_cm", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - # Informational output - print("\n" + "=" * 60) - print("HEAD OBJECT BENCHMARK (ASYNC)") - print("=" * 60) - print( - f"boto3 head_object: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity head_object (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - - print("=" * 60) - - -@pytest.mark.asyncio -async def test_create_bucket_perf_aio_cm(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity async for create_bucket. - - This benchmark tests the async implementation's create_bucket functionality. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_create_bucket_perf_aio_cm") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - session = get_session() - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - async with AsyncClient(**rustfs_server) as async_light_client: - iterations = 500 - bucket_prefix = "perf-bucket-create" - - # Warm-up to mitigate one-time costs - for i in range(10): - bucket = f"{bucket_prefix}-warmup-{i}" - - await boto_client.create_bucket(Bucket=bucket) - - for i in range(10): - bucket = f"{bucket_prefix}-warmup-light-{i}" - - await async_light_client.create_bucket(Bucket=bucket) - - async def run_boto(n: int): - for i in range(n): - bucket = f"{bucket_prefix}-boto-{i}" - - await boto_client.create_bucket(Bucket=bucket) - - async def run_custom(n: int): - for i in range(n): - bucket = f"{bucket_prefix}-custom-{i}" - - await async_light_client.create_bucket(Bucket=bucket) - - t_boto = await _timeit_async_helper(run_boto, iterations) - t_custom = await _timeit_async_helper(run_custom, iterations) - - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "create_bucket_aio_cm", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - # Informational output - print("\n" + "=" * 60) - print("CREATE BUCKET BENCHMARK (ASYNC)") - print("=" * 60) - print( - f"boto3 create_bucket: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity create_bucket (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - - print("=" * 60) - - -@pytest.mark.asyncio -async def test_delete_objects_perf_aio_cm(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity async for delete_objects. - - This benchmark tests the async implementation's delete_objects functionality. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_delete_objects_perf_aio_cm") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - bucket = "perf-delete-objects" - num_keys = 10 - - session = get_session() - async with session.create_client( - "s3", **rustfs_server, config=Config(signature_version="s3v4") - ) as boto_client: - async with AsyncClient(**rustfs_server) as async_light_client: - # Create the bucket for testing - await async_light_client.create_bucket(Bucket=bucket) - - iterations = 10 - - async def _populate(prefix: str): - keys = [f"{prefix}-{i}.txt" for i in range(num_keys)] - for k in keys: - await boto_client.put_object(Bucket=bucket, Key=k, Body=b"data") - return keys - - # Warm-up - for i in range(5): - keys = await _populate(f"warmup-boto-{i}") - await boto_client.delete_objects( - Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} - ) - - for i in range(5): - keys = await _populate(f"warmup-light-{i}") - await async_light_client.delete_objects( - Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} - ) - - async def run_boto(n: int): - for i in range(n): - keys = await _populate(f"bench-boto-{i}") - await boto_client.delete_objects( - Bucket=bucket, - Delete={"Objects": [{"Key": k} for k in keys]}, - ) - - async def run_custom(n: int): - for i in range(n): - keys = await _populate(f"bench-light-{i}") - await async_light_client.delete_objects( - Bucket=bucket, - Delete={"Objects": [{"Key": k} for k in keys]}, - ) - - t_boto = await _timeit_async_helper(run_boto, iterations) - t_custom = await _timeit_async_helper(run_custom, iterations) - - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "delete_objects_aio_cm", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print("\n" + "=" * 60) - print("DELETE OBJECTS BENCHMARK (ASYNC CM)") - print("=" * 60) - print( - f"boto3 delete_objects: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity delete_objects (async cm): {t_custom:.4f}s" - f" for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - - print("=" * 60) diff --git a/benchmark_tests/test_benchmark_cm.py b/benchmark_tests/test_benchmark_cm.py deleted file mode 100644 index af6b14a..0000000 --- a/benchmark_tests/test_benchmark_cm.py +++ /dev/null @@ -1,483 +0,0 @@ -from __future__ import annotations - -import json -import os -import random -import sys -from pathlib import Path - -import boto3 -import pytest -from botocore.client import Config - -from conftest import _timeit -from signurlarity import Client - - -def test_generate_presigned_post_perf_sync_cm(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for presigned POST. - - This is a non-failing, informational test: it prints timings and skips - if the signurlarity implementation is not available. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_generate_presigned_post_perf_sync_cm") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - rng = random.Random(42) # noqa: S311 - bucket = "perf-bucket" - key = "object.txt" - - boto_client = boto3.client("s3", **rustfs_server) - with Client(**rustfs_server) as light_client: - # Bucket creation ensures produced URLs are fully valid for the endpoint - # but is not part of the benchmark itself. - # boto_client.create_bucket(Bucket=bucket) - - # Minimal fields/conditions for a fair apples-to-apples comparison - fields = None - conditions = None - - iterations = 500 - - # Warm-up to mitigate one-time costs (imports, JIT-like caches, etc.) - for _ in range(50): - boto_client.generate_presigned_post( - Bucket=bucket, - Key=key, - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - try: - for _ in range(50): - light_client.generate_presigned_post( - Bucket=bucket, - Key=key, - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - except NotImplementedError: - pytest.skip( - "signurlarity.Client.generate_presigned_post not implemented; skipping perf comparison" - ) - - def run_boto(n: int): - for _ in range(n): - # Vary key slightly to avoid any internal memoization across loops - boto_client.generate_presigned_post( - Bucket=bucket, - Key=f"{key}-{rng.randint(0, 1_000_000)}", - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - - def run_light(n: int): - for _ in range(n): - light_client.generate_presigned_post( - Bucket=bucket, - Key=f"{key}-{rng.randint(0, 1_000_000)}", - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_light, iterations) - - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "generate_presigned_post_sync_cm", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print(results) - result_file.write_text(json.dumps(results, indent=2)) - - -def test_generate_presigned_url_perf_sync_cm(rustfs_server, test_results_dir): - """Compare performance of boto3 vs custom S3PresignedURLGenerator for presigned URL. - - This benchmark compares boto3's generate_presigned_url with the custom - implementation that has zero boto3 dependencies. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_generate_presigned_url_perf_sync_cm") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - rng = random.Random(42) # noqa: S311 - bucket = "perf-bucket" - key = "object.txt" - - # Extract region from endpoint_url - region = "us-east-1" - - boto_client = boto3.client( - "s3", - region_name=region, - **rustfs_server, - config=Config(signature_version="s3v4"), - ) - with Client(**rustfs_server) as light_client: - # custom_generator = S3PresignedURLGenerator( - # access_key=AWS_ACCESS_KEY_ID, secret_key=AWS_SECRET_ACCESS_KEY, region=region - # ) - - iterations = 500 - - # Warm-up to mitigate one-time costs - for _ in range(50): - boto_client.generate_presigned_url( - "get_object", Params={"Bucket": bucket, "Key": key}, ExpiresIn=60 - ) - - for _ in range(50): - light_client.generate_presigned_url( - "get_object", Params={"Bucket": bucket, "Key": key}, ExpiresIn=60 - ) - - def run_boto(n: int): - for _ in range(n): - # Vary key slightly to avoid any internal memoization - boto_client.generate_presigned_url( - "get_object", - Params={ - "Bucket": bucket, - "Key": f"{key}-{rng.randint(0, 1_000_000)}", - }, - ExpiresIn=60, - ) - - def run_custom(n: int): - for _ in range(n): - light_client.generate_presigned_url( - "get_object", - Params={ - "Bucket": bucket, - "Key": f"{key}-{rng.randint(0, 1_000_000)}", - }, - ExpiresIn=60, - ) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_custom, iterations) - - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "generate_presigned_url_sync_cm", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print(results) - result_file.write_text(json.dumps(results, indent=2)) - - -def test_head_bucket_perf_sync_cm(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for head_bucket. - - This benchmark compares boto3's head_bucket with the custom - implementation that has zero boto3 dependencies. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_head_bucket_perf_sync_cm") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - bucket = "perf-bucket" - - # Extract region from endpoint_url - region = "us-east-1" - - boto_client = boto3.client("s3", region_name=region, **rustfs_server) - with Client(**rustfs_server) as light_client: - # Create the bucket for testing - boto_client.create_bucket(Bucket=bucket) - - iterations = 500 - - # Warm-up to mitigate one-time costs - for _ in range(10): - boto_client.head_bucket(Bucket=bucket) - - for _ in range(10): - light_client.head_bucket(Bucket=bucket) - - def run_boto(n: int): - for _ in range(n): - boto_client.head_bucket(Bucket=bucket) - - def run_custom(n: int): - for _ in range(n): - light_client.head_bucket(Bucket=bucket) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_custom, iterations) - - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "head_bucket_sync_cm", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - result_file.write_text(json.dumps(results, indent=2)) - - -def test_head_object_perf_sync_cm(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for head_object. - - This benchmark compares boto3's head_object with the custom - implementation that has zero boto3 dependencies. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_head_object_perf_sync_cm") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - bucket = "perf-object" - key = "perf-object.txt" - - # Extract region from endpoint_url - region = "us-east-1" - - boto_client = boto3.client("s3", region_name=region, **rustfs_server) - with Client(**rustfs_server) as light_client: - # Create the bucket and object for testing - boto_client.create_bucket(Bucket=bucket) - boto_client.put_object( - Bucket=bucket, Key=key, Body=b"test data for head_object perf test" - ) - - iterations = 500 - - # Warm-up to mitigate one-time costs - for _ in range(10): - boto_client.head_object(Bucket=bucket, Key=key) - - for _ in range(10): - light_client.head_object(Bucket=bucket, Key=key) - - def run_boto(n: int): - for _ in range(n): - boto_client.head_object(Bucket=bucket, Key=key) - - def run_custom(n: int): - for _ in range(n): - light_client.head_object(Bucket=bucket, Key=key) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_custom, iterations) - - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "head_object_sync_cm", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - # Informational output - print("\n" + "=" * 60) - print("HEAD OBJECT BENCHMARK") - print("=" * 60) - print( - f"boto3 head_object: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity head_object: {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - - print("=" * 60) - - -def test_create_bucket_perf_sync_cm(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for create_bucket. - - This benchmark compares boto3's create_bucket with the signurlarity - implementation that uses httpx with AWS Signature V4. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_create_bucket_perf_sync_cm") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - boto_client = boto3.client("s3", **rustfs_server) - with Client(**rustfs_server) as light_client: - iterations = 500 - bucket_prefix = "perf-bucket-create" - - # Warm-up to mitigate one-time costs - for i in range(10): - bucket = f"{bucket_prefix}-warmup-{i}" - boto_client.create_bucket(Bucket=bucket) - boto_client.delete_bucket(Bucket=bucket) - - for i in range(10): - bucket = f"{bucket_prefix}-warmup-light-{i}" - light_client.create_bucket(Bucket=bucket) - boto_client.delete_bucket(Bucket=bucket) - - def run_boto(n: int): - for i in range(n): - bucket = f"{bucket_prefix}-boto-{i}" - boto_client.create_bucket(Bucket=bucket) - boto_client.delete_bucket(Bucket=bucket) - - def run_custom(n: int): - for i in range(n): - bucket = f"{bucket_prefix}-custom-{i}" - light_client.create_bucket(Bucket=bucket) - boto_client.delete_bucket(Bucket=bucket) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_custom, iterations) - - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "create_bucket_sync_cm", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - # Informational output - print("\n" + "=" * 60) - print("CREATE BUCKET BENCHMARK") - print("=" * 60) - print( - f"boto3 create_bucket: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity create_bucket: {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - - print("=" * 60) - - -def test_delete_objects_perf_sync_cm(rustfs_server, test_results_dir): - """Compare performance of boto3 vs signurlarity for delete_objects. - - This benchmark compares boto3's delete_objects with the signurlarity - implementation that uses httpx with AWS Signature V4. - """ - py_vers = sys.version_info - test_dir = test_results_dir / Path("test_delete_objects_perf_sync_cm") - os.makedirs(test_dir, exist_ok=True) - result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json") - - bucket = "perf-delete-objects" - num_keys = 10 - - boto_client = boto3.client("s3", **rustfs_server) - with Client(**rustfs_server) as light_client: - # Create the bucket for testing - boto_client.create_bucket(Bucket=bucket) - - iterations = 10 - - def _populate(prefix: str): - keys = [f"{prefix}-{i}.txt" for i in range(num_keys)] - for k in keys: - boto_client.put_object(Bucket=bucket, Key=k, Body=b"data") - return keys - - # Warm-up - for i in range(5): - keys = _populate(f"warmup-boto-{i}") - boto_client.delete_objects( - Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} - ) - - for i in range(5): - keys = _populate(f"warmup-light-{i}") - light_client.delete_objects( - Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} - ) - - def run_boto(n: int): - for i in range(n): - keys = _populate(f"bench-boto-{i}") - boto_client.delete_objects( - Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} - ) - - def run_custom(n: int): - for i in range(n): - keys = _populate(f"bench-light-{i}") - light_client.delete_objects( - Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]} - ) - - t_boto = _timeit(run_boto, iterations) - t_custom = _timeit(run_custom, iterations) - - results = { - "python_version": f"{py_vers.major}.{py_vers.minor}", - "tested_method": "delete_objects_sync_cm", - "iterations": iterations, - "boto_total": t_boto, - "signurlarity_total": t_custom, - "boto_ops": iterations / t_boto, - "signurlarity_ops": iterations / t_custom, - "speedup": t_boto / t_custom, - } - - print("\n" + "=" * 60) - print("DELETE OBJECTS BENCHMARK") - print("=" * 60) - print( - f"boto3 delete_objects: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)" - ) - print( - f"signurlarity delete_objects: {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)" - ) - if t_custom > 0: - speedup = t_boto / t_custom - print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x") - if speedup > 1: - print(f"✓ Signurlarity implementation is {speedup:.2f}x FASTER!") - else: - print(f"boto3 is {1 / speedup:.2f}x faster") - - result_file.write_text(json.dumps(results, indent=2)) - - print("=" * 60) diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..64cefaa --- /dev/null +++ b/conftest.py @@ -0,0 +1,434 @@ +"""Root conftest.py - Shared fixtures for all test directories. + +This file contains common fixtures used across: +- tests/ - Unit tests +- benchmark_tests/ - Performance benchmarks +- profiling_tests/ - Profiling tests +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import random +import signal +import subprocess +import time +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Generator + +import boto3 +import botocore +import pytest +from aiobotocore.session import get_session +from botocore.client import Config + +from signurlarity import Client +from signurlarity.aio import AsyncClient + +# ============================================================================= +# Constants +# ============================================================================= + +BUCKET_NAME = "test-bucket" +OTHER_BUCKET_NAME = "other-bucket" +MISSING_BUCKET_NAME = "missing-bucket" +INVALID_BUCKET_NAME = ".." + +CHECKSUM_ALGORITHM = "sha256" + +# Random number generator with fixed seed for reproducibility +rng = random.Random(1234) # noqa: S311 + + +# ============================================================================= +# Utility Functions +# ============================================================================= + + +def random_file(size_bytes: int) -> tuple[bytes, str]: + """Generate random file content and its SHA256 checksum.""" + file_content = rng.randbytes(size_bytes) + checksum = hashlib.sha256(file_content).hexdigest() + return file_content, checksum + + +def b16_to_b64(hex_string: str) -> str: + """Convert hexadecimal encoded data to base64 encoded data.""" + return base64.b64encode(base64.b16decode(hex_string.upper())).decode() + + +# ============================================================================= +# Pytest Configuration +# ============================================================================= + + +def pytest_addoption(parser): + """Add command line options for test result directories.""" + parser.addoption( + "--test-results-dir", + type=Path, + default=None, + help="Path to store the test results", + ) + + +# ============================================================================= +# Test Results Directory Fixture +# ============================================================================= + + +@pytest.fixture(scope="module") +def test_results_dir(request) -> Generator[Path, None, None]: + """Get the test results directory from command line or skip if not provided.""" + test_results_dir = request.config.getoption("--test-results-dir") + if test_results_dir is None: + pytest.skip( + "Requires a directory to store the test results with --test-results-dir" + ) + test_results_dir = test_results_dir.resolve() + yield test_results_dir + + +# ============================================================================= +# Server Fixtures +# ============================================================================= + + +@pytest.fixture(scope="module") +def moto_server(worker_id): + """Start the moto server in a separate thread and return the base URL. + + The mocking provided by moto doesn't play nicely with aiobotocore so we use + the server directly. See https://github.com/aio-libs/aiobotocore/issues/755 + """ + AWS_ACCESS_KEY_ID = "testing" + AWS_SECRET_ACCESS_KEY = "testing" # noqa: S105 + + from moto.server import ThreadedMotoServer + + port = 27132 + if worker_id != "master": + port += int(worker_id.replace("gw", "")) + 1 + server = ThreadedMotoServer(port=port) + server.start() + yield { + "endpoint_url": f"http://localhost:{port}", + "aws_access_key_id": AWS_ACCESS_KEY_ID, + "aws_secret_access_key": AWS_SECRET_ACCESS_KEY, + } + + server.stop() + + +@pytest.fixture(scope="module") +def rustfs_server(): + """Run a rustfs server.""" + AWS_ACCESS_KEY_ID = "rustfsadmin" + AWS_SECRET_ACCESS_KEY = "rustfsadmin" # noqa: S105 + + cmd = [ + "docker", + "run", + "-d", + "--rm", + "--name", + "rustfs_local", + "-p", + "9000:9000", + "-p", + "9001:9001", + "rustfs/rustfs:1.0.0-alpha.82", # return to latest when https://github.com/rustfs/rustfs/issues/1773 is fixed + "/data", + ] + + subprocess.run(cmd, check=True) # noqa: S603 + time.sleep(1) # Wait for server to start + yield { + "endpoint_url": "http://localhost:9000", + "aws_access_key_id": AWS_ACCESS_KEY_ID, + "aws_secret_access_key": AWS_SECRET_ACCESS_KEY, + } + cmd = ["docker", "stop", "rustfs_local"] + subprocess.run(cmd, check=True) # noqa: S603 + + +@pytest.fixture(scope="module") +def minio_server(): + """Run a minio server.""" + AWS_ACCESS_KEY_ID = "minioadmin" + AWS_SECRET_ACCESS_KEY = "minioadmin" # noqa: S105 + + cmd = [ + "docker", + "run", + "-d", + "--rm", + "--name", + "minio_local", + "-p", + "9100:9000", + "-p", + "9101:9001", + "-e", + "MINIO_ROOT_USER=minioadmin", + "-e", + "MINIO_ROOT_PASSWORD=minioadmin", + "minio/minio", + "server", + "/data", + ] + subprocess.run(cmd, check=True) # noqa: S603 + yield { + "endpoint_url": "http://localhost:9100", + "aws_access_key_id": AWS_ACCESS_KEY_ID, + "aws_secret_access_key": AWS_SECRET_ACCESS_KEY, + } + cmd = ["docker", "stop", "minio_local"] + subprocess.run(cmd, check=True) # noqa: S603 + + +@pytest.fixture(scope="module") +def seaweedfs_server(): + """Run a SeaweedFS server with S3 API enabled. + + Because it creates volumes on the fly, we have to upload a file + and wait for the initialization to be over, otherwise all the tests + fail. + """ + AWS_ACCESS_KEY_ID = "admin" + AWS_SECRET_ACCESS_KEY = "key" # noqa: S105 + + def check_volume_status(max_retries=10, retry_delay=5): + cmd = ["weed", "shell"] + # Use echo to send the command to weed shell + input_cmd = "cluster.status\n" + + for attempt in range(1, max_retries + 1): + try: + process = subprocess.Popen( # noqa: S603 + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + stdout, _stderr = process.communicate(input=input_cmd, timeout=15) + + # Check if "7 volume" is in the output + if "7 volume" in stdout: + print("Found '7 volume' in output!") + return + + print( + f"'7 volume' not found (attempt {attempt}/{max_retries}), " + f"retrying in {retry_delay} seconds..." + ) + except subprocess.TimeoutExpired: + process.kill() + stdout, _stderr = process.communicate() + print( + f"weed shell timed out (attempt {attempt}/{max_retries}), " + f"retrying in {retry_delay} seconds..." + ) + except Exception as exc: + print( + f"Error checking volume status (attempt {attempt}/{max_retries}): {exc}" + ) + + if attempt < max_retries: + time.sleep(retry_delay) + + raise RuntimeError( + f"SeaweedFS did not report '7 volume' after {max_retries} attempts" + ) + + with TemporaryDirectory() as tmp_dir: + os.mkdir(f"{tmp_dir}/seaweedfs") + with open(f"{tmp_dir}/seaweedfs_s3.json", "wt") as f: + json.dump( + { + "identities": [ + { + "name": "admin", + "credentials": [ + { + "accessKey": AWS_ACCESS_KEY_ID, + "secretKey": AWS_SECRET_ACCESS_KEY, + } + ], + "actions": ["Admin", "Read", "Write", "List", "Tagging"], + } + ] + }, + f, + ) + cmd = [ + "weed", + "-v", + "4", + "mini", + "-dir", + f"{tmp_dir}/seaweedfs", + "-s3.config", + f"{tmp_dir}/seaweedfs_s3.json", + ] + with open(f"{tmp_dir}/seaweedfs.log", "w") as log_file: + pid = None + try: + process = subprocess.Popen( # noqa: S603 + cmd, + stdout=log_file, + stderr=subprocess.STDOUT, # Redirect stderr to stdout + ) + + pid = process.pid + print(f"Process PID: {pid} Working Directory {tmp_dir}") + upload_cmd = [ + "weed", + "upload", + "-master", + "localhost:9333", + f"{tmp_dir}/seaweedfs.log", + ] + max_retries = 10 + retry_delay = 5 + + for attempt in range(1, max_retries + 1): + try: + subprocess.run( # noqa: S603 + upload_cmd, check=True, capture_output=True, text=True + ) + print("Upload successful!") + break + except subprocess.CalledProcessError as e: + if attempt >= max_retries: + raise RuntimeError( + f"Upload failed after {max_retries} attempts: {e.stderr}" + ) from e + print( + f"Upload failed (attempt {attempt}/{max_retries}), " + f"retrying in {retry_delay} seconds... (Error: {e.stderr})" + ) + time.sleep(retry_delay) + check_volume_status() + + yield { + "endpoint_url": "http://localhost:8333", + "aws_access_key_id": AWS_ACCESS_KEY_ID, + "aws_secret_access_key": AWS_SECRET_ACCESS_KEY, + } + except RuntimeError as e: + print(e) + log_file.flush() + print("=== SeaweedFS log start ===") + try: + with open( + f"{tmp_dir}/seaweedfs.log", + "rt", + encoding="utf-8", + errors="replace", + ) as read_log: + print(read_log.read()) + except OSError as log_error: + print(f"Failed to read SeaweedFS log file: {log_error}") + print("=== SeaweedFS log end ===") + raise + finally: + if pid: + os.kill(pid, signal.SIGKILL) + + +# ============================================================================= +# Client Fixtures (for unit tests) +# ============================================================================= + + +@pytest.fixture( + scope="function", + params=["minio_server", "moto_server", "rustfs_server", "seaweedfs_server"], +) +def s3_clients(request): + """S3 clients for synchronous tests with multiple server backends. + + This fixture can be used to test S3 interactions using different + backends (moto, minio, rustfs). Returns both boto3 and signurlarity clients. + """ + s3_server_fixture = request.param + s3_server = request.getfixturevalue(s3_server_fixture) + boto_client = boto3.client( + "s3", **s3_server, config=Config(signature_version="s3v4") + ) + light_client = Client(**s3_server) + + try: + boto_client.head_bucket(Bucket=BUCKET_NAME) + except botocore.exceptions.ClientError as exx: + if exx.response["Error"]["Code"] == "404": + boto_client.create_bucket(Bucket=BUCKET_NAME) + yield boto_client, light_client + light_client.close() + + +@pytest.fixture( + scope="function", + params=["minio_server", "moto_server", "rustfs_server", "seaweedfs_server"], +) +async def s3_clients_aio(request): + """S3 clients for asynchronous tests with multiple server backends. + + This fixture can be used to test async S3 interactions using different + backends (moto, minio, rustfs). Returns both aiobotocore and signurlarity async clients. + """ + s3_server_fixture = request.param + s3_server = request.getfixturevalue(s3_server_fixture) + AIO_BUCKET_NAME = f"{BUCKET_NAME}-aio" + + session = get_session() + async with session.create_client( + "s3", + endpoint_url=s3_server["endpoint_url"], + aws_access_key_id=s3_server["aws_access_key_id"], + aws_secret_access_key=s3_server["aws_secret_access_key"], + config=Config(signature_version="s3v4"), + ) as boto_client: + async_light_client = AsyncClient(**s3_server) + + try: + await boto_client.head_bucket(Bucket=AIO_BUCKET_NAME) + except Exception: + await boto_client.create_bucket(Bucket=AIO_BUCKET_NAME) + + yield boto_client, async_light_client + await async_light_client.close() + + +# ============================================================================= +# Timing Utilities (for benchmark tests) +# ============================================================================= + + +def _timeit(fn, iterations: int) -> float: + """Measure execution time of a synchronous function.""" + start = time.perf_counter() + fn(iterations) + return time.perf_counter() - start + + +def _timeit_async(fn, iterations: int) -> float: + """Measure execution time of an async function (from sync context).""" + import asyncio + + start = time.perf_counter() + asyncio.run(fn(iterations)) + return time.perf_counter() - start + + +async def _timeit_async_helper(fn, iterations: int) -> float: + """Measure the execution time of an async function within async context.""" + start = time.perf_counter() + await fn(iterations) + return time.perf_counter() - start diff --git a/profiling_tests/__init__.py b/profiling_tests/__init__.py new file mode 100644 index 0000000..a34a45c --- /dev/null +++ b/profiling_tests/__init__.py @@ -0,0 +1 @@ +# Profiling tests package diff --git a/profiling_tests/conftest.py b/profiling_tests/conftest.py index 7ffa0bb..10c8002 100644 --- a/profiling_tests/conftest.py +++ b/profiling_tests/conftest.py @@ -1,25 +1,30 @@ -from __future__ import annotations +"""Test fixtures for the profiling_tests directory. -from pathlib import Path +This file re-exports fixtures from the root conftest.py and can contain +profiling-specific fixtures that are not needed elsewhere. -import pytest +NOTE: pytest_addoption is NOT re-exported here because it should only be +registered once (in the root conftest.py). +""" +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path -def pytest_addoption(parser): - parser.addoption( - "--test-results-dir", - type=Path, - default=None, - help="Path to store the perf test results", - ) +# Add project root to path so we can import root_conftest_module +_project_root = str(Path(__file__).parent.parent) +if _project_root not in sys.path: + sys.path.insert(0, _project_root) +# Import root conftest as a module with a unique name +_root_conftest_path = str(Path(_project_root) / "conftest.py") +spec = importlib.util.spec_from_file_location( + "root_conftest_module", _root_conftest_path +) +root_conftest_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(root_conftest_module) -@pytest.fixture(scope="module") -def test_results_dir(request) -> Path: - test_results_dir = request.config.getoption("--test-results-dir") - if test_results_dir is None: - pytest.skip( - "Requires a directory to store the test results with --test-results-dir" - ) - test_results_dir = test_results_dir.resolve() - yield test_results_dir +# Re-export fixtures (but NOT pytest_addoption hook which should only be registered once) +test_results_dir = root_conftest_module.test_results_dir diff --git a/pyproject.toml b/pyproject.toml index 921164f..7904eb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ testing = [ "pytest-asyncio", "aiobotocore>=2.15", "botocore>=1.35", + "boto3>=1.35", "moto[server]", "httpx", "pytest-xdist", @@ -52,6 +53,9 @@ where = ["src"] pythonpath = ["src"] testpaths = ["tests"] asyncio_mode = "auto" +markers = [ + "moto: mark test as using moto server (only runs with moto backend)", +] [tool.ruff] diff --git a/src/signurlarity/presigner.py b/src/signurlarity/presigner.py index b77058b..7e39fc5 100644 --- a/src/signurlarity/presigner.py +++ b/src/signurlarity/presigner.py @@ -544,6 +544,7 @@ def generate_presigned_post( {"x-amz-algorithm": "AWS4-HMAC-SHA256"}, {"x-amz-credential": credential}, {"x-amz-date": amz_date}, + {"X-Amz-Date": amz_date}, ] ) diff --git a/tests/__init__.py b/tests/__init__.py index 73d90cd..d4839a6 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1 @@ -# Test package initialization +# Tests package diff --git a/tests/conftest.py b/tests/conftest.py index 3fe1888..730e28d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,352 +1,32 @@ -"""Shared test fixtures for synchronous and asynchronous tests.""" +"""Test fixtures for the tests directory. -from __future__ import annotations - -import base64 -import hashlib -import json -import os -import random -import signal -import subprocess -import time -from tempfile import TemporaryDirectory - -import boto3 -import botocore -import pytest -from aiobotocore.session import get_session -from botocore.client import Config - -from signurlarity import Client -from signurlarity.aio import AsyncClient - -# Constants -BUCKET_NAME = "test-bucket" -OTHER_BUCKET_NAME = "other-bucket" -MISSING_BUCKET_NAME = "missing-bucket" -INVALID_BUCKET_NAME = ".." - -CHECKSUM_ALGORITHM = "sha256" - -rng = random.Random(1234) # noqa: S311 - - -# Utility functions -def random_file(size_bytes: int): - """Generate random file content and its SHA256 checksum.""" - file_content = rng.randbytes(size_bytes) - checksum = hashlib.sha256(file_content).hexdigest() - return file_content, checksum - - -def b16_to_b64(hex_string: str) -> str: - """Convert hexadecimal encoded data to base64 encoded data.""" - return base64.b64encode(base64.b16decode(hex_string.upper())).decode() - - -# Server fixtures -@pytest.fixture(scope="module") -def moto_server(worker_id): - """Start the moto server in a separate thread and return the base URL. - - The mocking provided by moto doesn't play nicely with aiobotocore so we use - the server directly. See https://github.com/aio-libs/aiobotocore/issues/755 - """ - AWS_ACCESS_KEY_ID = "testing" - AWS_SECRET_ACCESS_KEY = "testing" # noqa: S105 - - from moto.server import ThreadedMotoServer - - port = 27132 - if worker_id != "master": - port += int(worker_id.replace("gw", "")) + 1 - server = ThreadedMotoServer(port=port) - server.start() - yield { - "endpoint_url": f"http://localhost:{port}", - "aws_access_key_id": AWS_ACCESS_KEY_ID, - "aws_secret_access_key": AWS_SECRET_ACCESS_KEY, - } - - server.stop() - - -@pytest.fixture(scope="module") -def rustfs_server(): - """Run a rustfs server.""" - AWS_ACCESS_KEY_ID = "rustfsadmin" - AWS_SECRET_ACCESS_KEY = "rustfsadmin" # noqa: S105 - import subprocess - - cmd = [ - "docker", - "run", - "-d", - "--rm", - "--name", - "rustfs_local", - "-p", - "9000:9000", - "-p", - "9001:9001", - "rustfs/rustfs:1.0.0-alpha.82", # return to latest when https://github.com/rustfs/rustfs/issues/1773 is fixed - "/data", - ] - # print(shlex.join(cmd)) - - subprocess.run(cmd, check=True) # noqa: S603 - yield { - "endpoint_url": "http://localhost:9000", - "aws_access_key_id": AWS_ACCESS_KEY_ID, - "aws_secret_access_key": AWS_SECRET_ACCESS_KEY, - } - cmd = ["docker", "stop", "rustfs_local"] - subprocess.run(cmd, check=True) # noqa: S603 - - -@pytest.fixture(scope="module") -def minio_server(): - """Run a minio server.""" - AWS_ACCESS_KEY_ID = "minioadmin" - AWS_SECRET_ACCESS_KEY = "minioadmin" # noqa: S105 - import subprocess +This file re-exports constants and utilities from the root conftest.py. +Pytest fixtures defined in root conftest.py are automatically available. +""" - cmd = [ - "docker", - "run", - "-d", - "--rm", - "--name", - "minio_local", - "-p", - "9100:9000", - "-p", - "9101:9001", - "-e", - "MINIO_ROOT_USER=minioadmin", - "-e", - "MINIO_ROOT_PASSWORD=minioadmin", - "minio/minio", - "server", - "/data", - ] - subprocess.run(cmd, check=True) # noqa: S603 - yield { - "endpoint_url": "http://localhost:9100", - "aws_access_key_id": AWS_ACCESS_KEY_ID, - "aws_secret_access_key": AWS_SECRET_ACCESS_KEY, - } - cmd = ["docker", "stop", "minio_local"] - - subprocess.run(cmd, check=True) # noqa: S603 - - -@pytest.fixture(scope="module") -def seaweedfs_server(): - """Run a SeaweedFS server with S3 API enabled. - - Because it creates volumes on the fly, we have to upload a file - and wait for the initialization to be over, otherwise all the tests - fail. - """ - AWS_ACCESS_KEY_ID = "admin" - AWS_SECRET_ACCESS_KEY = "key" # noqa: S105 - - def check_volume_status(max_retries=10, retry_delay=5): - cmd = ["weed", "shell"] - # Use echo to send the command to weed shell - input_cmd = "cluster.status\n" - - for attempt in range(1, max_retries + 1): - try: - process = subprocess.Popen( # noqa: S603 - cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - stdout, _stderr = process.communicate(input=input_cmd, timeout=15) - - # Check if "7 volume" is in the output - if "7 volume" in stdout: - print("Found '7 volume' in output!") - return - - print( - f"'7 volume' not found (attempt {attempt}/{max_retries}), " - f"retrying in {retry_delay} seconds..." - ) - except subprocess.TimeoutExpired: - process.kill() - stdout, _stderr = process.communicate() - print( - f"weed shell timed out (attempt {attempt}/{max_retries}), " - f"retrying in {retry_delay} seconds..." - ) - except Exception as exc: - print( - f"Error checking volume status (attempt {attempt}/{max_retries}): {exc}" - ) - - if attempt < max_retries: - time.sleep(retry_delay) - - raise RuntimeError( - f"SeaweedFS did not report '7 volume' after {max_retries} attempts" - ) - - with TemporaryDirectory() as tmp_dir: - os.mkdir(f"{tmp_dir}/seaweedfs") - with open(f"{tmp_dir}/seaweedfs_s3.json", "wt") as f: - json.dump( - { - "identities": [ - { - "name": "admin", - "credentials": [ - { - "accessKey": AWS_ACCESS_KEY_ID, - "secretKey": AWS_SECRET_ACCESS_KEY, - } - ], - "actions": ["Admin", "Read", "Write", "List", "Tagging"], - } - ] - }, - f, - ) - cmd = [ - "weed", - "mini", - "-dir", - f"{tmp_dir}/seaweedfs", - "-s3.config", - f"{tmp_dir}/seaweedfs_s3.json", - ] - with open(f"{tmp_dir}/seaweedfs.log", "w") as log_file: - pid = None - try: - process = subprocess.Popen( # noqa: S603 - cmd, - stdout=log_file, - stderr=subprocess.STDOUT, # Redirect stderr to stdout - ) - - pid = process.pid - print(f"Process PID: {pid} Working Directory {tmp_dir}") - upload_cmd = [ - "weed", - "upload", - "-master", - "localhost:9333", - f"{tmp_dir}/seaweedfs.log", - ] - max_retries = 10 - retry_delay = 5 - - for attempt in range(1, max_retries + 1): - try: - subprocess.run( # noqa: S603 - upload_cmd, check=True, capture_output=True, text=True - ) - print("Upload successful!") - break - except subprocess.CalledProcessError as e: - if attempt >= max_retries: - raise RuntimeError( - f"Upload failed after {max_retries} attempts: {e.stderr}" - ) from e - print( - f"Upload failed (attempt {attempt}/{max_retries}), " - f"retrying in {retry_delay} seconds... (Error: {e.stderr})" - ) - time.sleep(retry_delay) - check_volume_status() - - yield { - "endpoint_url": "http://localhost:8333", - "aws_access_key_id": AWS_ACCESS_KEY_ID, - "aws_secret_access_key": AWS_SECRET_ACCESS_KEY, - } - except RuntimeError as e: - print(e) - log_file.flush() - print("=== SeaweedFS log start ===") - try: - with open( - f"{tmp_dir}/seaweedfs.log", - "rt", - encoding="utf-8", - errors="replace", - ) as read_log: - print(read_log.read()) - except OSError as log_error: - print(f"Failed to read SeaweedFS log file: {log_error}") - print("=== SeaweedFS log end ===") - raise - finally: - if pid: - os.kill(pid, signal.SIGKILL) - - -# Synchronous client fixtures -@pytest.fixture( - scope="function", - params=["minio_server", "moto_server", "rustfs_server", "seaweedfs_server"], -) -def s3_clients(request): - """S3 clients for synchronous tests with multiple server backends. - - This fixture can be used to test S3 interactions using different - backends (moto, minio, rustfs). Returns both boto3 and signurlarity clients. - """ - s3_server_fixture = request.param - s3_server = request.getfixturevalue(s3_server_fixture) - boto_client = boto3.client( - "s3", **s3_server, config=Config(signature_version="s3v4") - ) - light_client = Client(**s3_server) +from __future__ import annotations - try: - boto_client.head_bucket(Bucket=BUCKET_NAME) - except botocore.exceptions.ClientError as exx: - if exx.response["Error"]["Code"] == "404": - boto_client.create_bucket(Bucket=BUCKET_NAME) - yield boto_client, light_client - light_client.close() +import importlib.util +import sys +from pathlib import Path +# Add project root to path so we can import root_conftest_module +_project_root = str(Path(__file__).parent.parent) +if _project_root not in sys.path: + sys.path.insert(0, _project_root) -# Asynchronous client fixtures -@pytest.fixture( - scope="function", - params=["minio_server", "moto_server", "rustfs_server", "seaweedfs_server"], +# Import root conftest as a module with a unique name +_root_conftest_path = str(Path(_project_root) / "conftest.py") +spec = importlib.util.spec_from_file_location( + "root_conftest_module", _root_conftest_path ) -async def s3_clients_aio(request): - """S3 clients for asynchronous tests with multiple server backends. - - This fixture can be used to test async S3 interactions using different - backends (moto, minio, rustfs). Returns both aiobotocore and signurlarity async clients. - """ - s3_server_fixture = request.param - s3_server = request.getfixturevalue(s3_server_fixture) - AIO_BUCKET_NAME = f"{BUCKET_NAME}-aio" - - session = get_session() - async with session.create_client( - "s3", - endpoint_url=s3_server["endpoint_url"], - aws_access_key_id=s3_server["aws_access_key_id"], - aws_secret_access_key=s3_server["aws_secret_access_key"], - config=Config(signature_version="s3v4"), - ) as boto_client: - async_light_client = AsyncClient(**s3_server) - - try: - await boto_client.head_bucket(Bucket=AIO_BUCKET_NAME) - except Exception: - await boto_client.create_bucket(Bucket=AIO_BUCKET_NAME) - - yield boto_client, async_light_client - await async_light_client.close() +root_conftest_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(root_conftest_module) + +# Re-export constants and utilities (fixtures are automatically available via pytest) +BUCKET_NAME = root_conftest_module.BUCKET_NAME +CHECKSUM_ALGORITHM = root_conftest_module.CHECKSUM_ALGORITHM +MISSING_BUCKET_NAME = root_conftest_module.MISSING_BUCKET_NAME +OTHER_BUCKET_NAME = root_conftest_module.OTHER_BUCKET_NAME +b16_to_b64 = root_conftest_module.b16_to_b64 +random_file = root_conftest_module.random_file diff --git a/tests/test_client.py b/tests/test_client.py deleted file mode 100644 index b568741..0000000 --- a/tests/test_client.py +++ /dev/null @@ -1,561 +0,0 @@ -from __future__ import annotations - -import logging -import tempfile - -import botocore -import httpx -import pytest - -from signurlarity.exceptions import NoSuchBucketError - -from .conftest import ( - BUCKET_NAME, - CHECKSUM_ALGORITHM, - MISSING_BUCKET_NAME, - OTHER_BUCKET_NAME, - b16_to_b64, - random_file, -) - -logging.basicConfig( - format="%(levelname)s [%(asctime)s] %(name)s - %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - level=logging.DEBUG, -) - - -def test_create_bucket(s3_clients): - boto_client, light_client = s3_clients - with pytest.raises(botocore.exceptions.ClientError): - boto_client.head_bucket(Bucket=OTHER_BUCKET_NAME) - light_client.create_bucket(Bucket=OTHER_BUCKET_NAME) - boto_client.head_bucket(Bucket=OTHER_BUCKET_NAME) - - -def test_head_bucket_exists(s3_clients): - """Test that head_bucket succeeds for an existing bucket.""" - _boto_client, light_client = s3_clients - response = light_client.head_bucket(Bucket=BUCKET_NAME) - - # Verify response structure - assert "BucketRegion" in response - assert "ResponseMetadata" in response - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - - -def test_head_bucket_not_found(s3_clients): - """Test that head_bucket raises NoSuchBucketError for non-existent bucket.""" - _boto_client, light_client = s3_clients - with pytest.raises(NoSuchBucketError): - light_client.head_bucket(Bucket=MISSING_BUCKET_NAME) - - -def test_head_object_exists(s3_clients): - """Test that head_object succeeds for an existing object.""" - boto_client, light_client = s3_clients - - # First, create an object - file_content = b"test content for head_object" - key = "test_file.txt" - boto_client.put_object( - Body=file_content, Bucket=BUCKET_NAME, Key=key, ContentType="text/plain" - ) - - # Now test head_object - response = light_client.head_object(Bucket=BUCKET_NAME, Key=key) - - # Verify response structure - assert "ContentLength" in response - assert "ETag" in response - assert "LastModified" in response - assert "ResponseMetadata" in response - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - assert response["ContentLength"] == len(file_content) - assert response["ContentType"] == "text/plain" - - -def test_head_object_not_found(s3_clients): - """Test that head_object raises PresignError for non-existent object.""" - _boto_client, light_client = s3_clients - from signurlarity.exceptions import PresignError - - with pytest.raises(PresignError): - light_client.head_object(Bucket=BUCKET_NAME, Key="nonexistent_key.txt") - - -def test_head_object_missing_key_param(s3_clients): - """Test that head_object raises PresignError when Key parameter is missing.""" - _boto_client, light_client = s3_clients - from signurlarity.exceptions import PresignError - - with pytest.raises(PresignError): - light_client.head_object(Bucket=BUCKET_NAME, Key="") - - -def test_head_object_missing_bucket_param(s3_clients): - """Test that head_object raises PresignError when Bucket parameter is missing.""" - _boto_client, light_client = s3_clients - from signurlarity.exceptions import PresignError - - with pytest.raises(PresignError): - light_client.head_object(Bucket="", Key="some_key.txt") - - -# def test_head_bucket_missing_bucket_param(s3_clients): -# """Test that head_bucket raises error when Bucket parameter is missing.""" -# _boto_client, light_client = s3_clients -# with pytest.raises(Exception): # PresignError or similar -# light_client.head_bucket(Bucket="") - - -def test_generate_presigned_post(s3_clients): - """Upload files using post presigned URLs. - - Get a pre-signed URL with our client, upload with httpx - check it exists with boto. - """ - boto_client, light_client = s3_clients - - file_content, checksum = random_file(128) - key = f"{checksum}.dat" - size = len(file_content) - - fields = { - "x-amz-checksum-algorithm": CHECKSUM_ALGORITHM, - f"x-amz-checksum-{CHECKSUM_ALGORITHM}": b16_to_b64(checksum), - } - conditions = [["content-length-range", size, size]] + [ - {k: v} for k, v in fields.items() - ] - - upload_info = light_client.generate_presigned_post( - Bucket=BUCKET_NAME, - Key=key, - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - - with httpx.Client() as client: - r = client.post( - upload_info["url"], - data=upload_info["fields"], - files={"file": file_content}, - ) - - assert r.status_code in (200, 204), r.text - boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - - -def test_generate_presigned_url(s3_clients, caplog): - """Get a pre-signed URL with our client, upload with httpx, check it exists with boto.""" - caplog.set_level(logging.DEBUG, logger="httpx") - caplog.set_level(logging.DEBUG, logger="httpcore") - - boto_client, light_client = s3_clients - - file_content, checksum = random_file(128) - key = f"{checksum}.dat" - - response = boto_client.put_object( - Body=file_content, Bucket=BUCKET_NAME, Key=key, Metadata={"Checksum": checksum} - ) - - presigned_url = light_client.generate_presigned_url( - ClientMethod="get_object", - Params={"Bucket": BUCKET_NAME, "Key": key}, - ExpiresIn=3600, - ) - - with tempfile.TemporaryFile(mode="w+b") as fh: - with httpx.Client() as http_client: - response = http_client.get(presigned_url) - response.raise_for_status() - for chunk in response.iter_bytes(): - fh.write(chunk) - - -def test_put_object(s3_clients): - """Test that put_object uploads bytes to a bucket.""" - boto_client, light_client = s3_clients - - file_content = b"hello from put_object" - key = "put-object-test.txt" - - response = light_client.put_object( - Bucket=BUCKET_NAME, - Key=key, - Body=file_content, - ContentType="text/plain", - ) - - assert "ETag" in response - assert "ResponseMetadata" in response - assert response["ResponseMetadata"]["HTTPStatusCode"] in (200, 201) - - # Verify via boto - head = boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - assert head["ContentLength"] == len(file_content) - - -def test_put_object_with_metadata(s3_clients): - """Test that put_object stores metadata on the object.""" - boto_client, light_client = s3_clients - - file_content = b"data with metadata" - key = "put-object-meta-test.txt" - - light_client.put_object( - Bucket=BUCKET_NAME, - Key=key, - Body=file_content, - Metadata={"author": "test", "version": "1"}, - ) - - head = boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - assert head["Metadata"].get("author") == "test" - assert head["Metadata"].get("version") == "1" - - -def test_put_object_missing_bucket(s3_clients): - """Test that put_object raises PresignError when Bucket is missing.""" - from signurlarity.exceptions import PresignError - - _boto_client, light_client = s3_clients - with pytest.raises(PresignError): - light_client.put_object(Bucket="", Key="key.txt", Body=b"data") - - -def test_put_object_missing_key(s3_clients): - """Test that put_object raises PresignError when Key is missing.""" - from signurlarity.exceptions import PresignError - - _boto_client, light_client = s3_clients - with pytest.raises(PresignError): - light_client.put_object(Bucket=BUCKET_NAME, Key="", Body=b"data") - - -def test_list_objects_empty(s3_clients): - """Test list_objects on a bucket with no matching prefix.""" - _boto_client, light_client = s3_clients - - response = light_client.list_objects( - Bucket=BUCKET_NAME, Prefix="list-objects-nonexistent-prefix/" - ) - - assert "ResponseMetadata" in response - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - assert response["Contents"] == [] - assert response["IsTruncated"] is False - - -def test_list_objects(s3_clients): - """Test list_objects returns uploaded objects.""" - boto_client, light_client = s3_clients - - keys = ["list-test/a.txt", "list-test/b.txt", "list-test/c.txt"] - for key in keys: - boto_client.put_object(Body=b"data", Bucket=BUCKET_NAME, Key=key) - - response = light_client.list_objects(Bucket=BUCKET_NAME, Prefix="list-test/") - - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - returned_keys = {obj["Key"] for obj in response["Contents"]} - assert returned_keys.issuperset(set(keys)) - for obj in response["Contents"]: - assert "Key" in obj - assert "ETag" in obj - assert "Size" in obj - assert "LastModified" in obj - - -def test_list_objects_with_delimiter(s3_clients): - """Test list_objects with delimiter groups common prefixes.""" - boto_client, light_client = s3_clients - - keys = ["delim-test/dir1/file.txt", "delim-test/dir2/file.txt"] - for key in keys: - boto_client.put_object(Body=b"data", Bucket=BUCKET_NAME, Key=key) - - response = light_client.list_objects( - Bucket=BUCKET_NAME, Prefix="delim-test/", Delimiter="/" - ) - - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - assert "CommonPrefixes" in response - prefixes = {cp["Prefix"] for cp in response["CommonPrefixes"]} - assert "delim-test/dir1/" in prefixes - assert "delim-test/dir2/" in prefixes - - -def test_list_objects_missing_bucket(s3_clients): - """Test that list_objects raises PresignError when Bucket is missing.""" - from signurlarity.exceptions import PresignError - - _boto_client, light_client = s3_clients - with pytest.raises(PresignError): - light_client.list_objects(Bucket="") - - -def test_copy_object(s3_clients): - """Test that copy_object copies an object using a string CopySource.""" - boto_client, light_client = s3_clients - - src_key = "copy-src.txt" - dst_key = "copy-dst.txt" - boto_client.put_object(Body=b"copy me", Bucket=BUCKET_NAME, Key=src_key) - - response = light_client.copy_object( - Bucket=BUCKET_NAME, - Key=dst_key, - CopySource=f"{BUCKET_NAME}/{src_key}", - ) - - assert "CopyObjectResult" in response - assert "ETag" in response["CopyObjectResult"] - assert "ResponseMetadata" in response - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - - # Verify destination exists - head = boto_client.head_object(Bucket=BUCKET_NAME, Key=dst_key) - assert head["ContentLength"] == len(b"copy me") - - -def test_copy_object_dict_source(s3_clients): - """Test that copy_object works with a dict CopySource.""" - boto_client, light_client = s3_clients - - src_key = "copy-src-dict.txt" - dst_key = "copy-dst-dict.txt" - boto_client.put_object(Body=b"dict source", Bucket=BUCKET_NAME, Key=src_key) - - response = light_client.copy_object( - Bucket=BUCKET_NAME, - Key=dst_key, - CopySource={"Bucket": BUCKET_NAME, "Key": src_key}, - ) - - assert "CopyObjectResult" in response - boto_client.head_object(Bucket=BUCKET_NAME, Key=dst_key) - - -def test_copy_object_missing_bucket(s3_clients): - """Test that copy_object raises PresignError when Bucket is missing.""" - from signurlarity.exceptions import PresignError - - _boto_client, light_client = s3_clients - with pytest.raises(PresignError): - light_client.copy_object( - Bucket="", Key="dst.txt", CopySource=f"{BUCKET_NAME}/src.txt" - ) - - -def test_copy_object_missing_copy_source(s3_clients): - """Test that copy_object raises PresignError when CopySource is missing.""" - from signurlarity.exceptions import PresignError - - _boto_client, light_client = s3_clients - with pytest.raises(PresignError): - light_client.copy_object(Bucket=BUCKET_NAME, Key="dst.txt", CopySource="") - - -def test_upload_file(s3_clients, tmp_path): - """Test that upload_file uploads a local file to S3.""" - boto_client, light_client = s3_clients - - content = b"file content to upload" - local_file = tmp_path / "upload_test.txt" - local_file.write_bytes(content) - - key = "upload-file-test.txt" - result = light_client.upload_file( - Filename=str(local_file), - Bucket=BUCKET_NAME, - Key=key, - ) - - assert result is None - - head = boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - assert head["ContentLength"] == len(content) - - -def test_upload_file_with_extra_args(s3_clients, tmp_path): - """Test that upload_file forwards ExtraArgs to put_object.""" - boto_client, light_client = s3_clients - - content = b"pdf content" - local_file = tmp_path / "report.pdf" - local_file.write_bytes(content) - - key = "upload-file-extra-args.pdf" - light_client.upload_file( - Filename=str(local_file), - Bucket=BUCKET_NAME, - Key=key, - ExtraArgs={"ContentType": "application/pdf"}, - ) - - head = boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - assert head["ContentType"] == "application/pdf" - assert head["ContentLength"] == len(content) - - -# Only Moto exposes the correct ACL api -@pytest.mark.parametrize( - "s3_clients", - [pytest.param("moto_server", marks=pytest.mark.moto)], - indirect=True, -) -def test_upload_file_with_acl_extra_args(s3_clients, tmp_path): - """Test that upload_file applies ACL from ExtraArgs (moto only).""" - boto_client, light_client = s3_clients - - content = b"acl content" - local_file = tmp_path / "acl.txt" - local_file.write_bytes(content) - - key = "upload-file-acl.txt" - light_client.upload_file( - Filename=str(local_file), - Bucket=BUCKET_NAME, - Key=key, - ExtraArgs={"ACL": "public-read"}, - ) - - acl = boto_client.get_object_acl(Bucket=BUCKET_NAME, Key=key) - grants = acl.get("Grants", []) - assert any( - grant.get("Permission") == "READ" - and grant.get("Grantee", {}).get("URI") - == "http://acs.amazonaws.com/groups/global/AllUsers" - for grant in grants - ), grants - - -def test_upload_file_missing_file(s3_clients): - """Test that upload_file raises OSError for a non-existent file.""" - _boto_client, light_client = s3_clients - with pytest.raises(OSError): - light_client.upload_file( - Filename="/nonexistent/path/file.txt", - Bucket=BUCKET_NAME, - Key="key.txt", - ) - - -def test_delete_objects(s3_clients): - """Test that delete_objects deletes multiple objects.""" - boto_client, light_client = s3_clients - - # Create some objects using boto - keys = ["delete-test-1.txt", "delete-test-2.txt", "delete-test-3.txt"] - for key in keys: - boto_client.put_object(Body=b"test content", Bucket=BUCKET_NAME, Key=key) - - # Verify objects exist - for key in keys: - boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - - # Delete objects using our client - response = light_client.delete_objects( - Bucket=BUCKET_NAME, - Delete={"Objects": [{"Key": k} for k in keys]}, - ) - - # Verify response structure - assert "ResponseMetadata" in response - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - assert "Deleted" in response - deleted_keys = {d["Key"] for d in response["Deleted"]} - assert deleted_keys == set(keys) - - # Verify objects are actually gone - for key in keys: - with pytest.raises(botocore.exceptions.ClientError): - boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - - -def test_delete_objects_quiet(s3_clients): - """Test that delete_objects with Quiet=True returns no Deleted list.""" - boto_client, light_client = s3_clients - - # Create objects - keys = ["delete-quiet-1.txt", "delete-quiet-2.txt"] - for key in keys: - boto_client.put_object(Body=b"test content", Bucket=BUCKET_NAME, Key=key) - - # Delete with Quiet=True - response = light_client.delete_objects( - Bucket=BUCKET_NAME, - Delete={"Objects": [{"Key": k} for k in keys], "Quiet": True}, - ) - - assert "ResponseMetadata" in response - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - # In quiet mode, only errors are reported - assert "Errors" not in response - - # Verify objects are actually gone - for key in keys: - with pytest.raises(botocore.exceptions.ClientError): - boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - - -def test_delete_objects_missing_bucket(s3_clients): - """Test that delete_objects raises PresignError for missing Bucket.""" - _boto_client, light_client = s3_clients - from signurlarity.exceptions import PresignError - - with pytest.raises(PresignError): - light_client.delete_objects( - Bucket="", - Delete={"Objects": [{"Key": "test.txt"}]}, - ) - - -def test_delete_objects_missing_objects(s3_clients): - """Test that delete_objects raises PresignError for missing Objects.""" - _boto_client, light_client = s3_clients - from signurlarity.exceptions import PresignError - - with pytest.raises(PresignError): - light_client.delete_objects(Bucket=BUCKET_NAME, Delete={}) - - with pytest.raises(PresignError): - light_client.delete_objects( - Bucket=BUCKET_NAME, - Delete={"Objects": []}, - ) - - -# @pytest.fixture() -# def fix_1(): -# print("entering fix 1") -# yield "fix 1" -# print("finishing fix 1") - - -# @pytest.fixture() -# def fix_2(worker_id): -# print(f"entering fix 2 {worker_id}") -# yield "fix 2" -# print("finishing fix 2") - - -# @pytest.fixture(params=["fix_1", "fix_2"]) -# def fix(request): -# server_fixture = request.param -# srv_fixt = request.getfixturevalue(server_fixture) -# print("entering fix") -# client_param = server_fixture -# # breakpoint() -# print(f"GOT {client_param}") -# yield client_param - -# print("finishing fix") - - -# def test_my_fix(fix): -# print(f"testing {fix}") diff --git a/tests/test_client_aio.py b/tests/test_client_aio.py deleted file mode 100644 index fae82d1..0000000 --- a/tests/test_client_aio.py +++ /dev/null @@ -1,571 +0,0 @@ -from __future__ import annotations - -import logging -import tempfile - -import httpx -import pytest -from botocore.errorfactory import ClientError - -from signurlarity.exceptions import NoSuchBucketError - -from .conftest import ( - BUCKET_NAME as BASE_BUCKET_NAME, -) -from .conftest import ( - CHECKSUM_ALGORITHM, - b16_to_b64, - random_file, -) -from .conftest import ( - MISSING_BUCKET_NAME as BASE_MISSING_BUCKET_NAME, -) -from .conftest import ( - OTHER_BUCKET_NAME as BASE_OTHER_BUCKET_NAME, -) - -logging.basicConfig( - format="%(levelname)s [%(asctime)s] %(name)s - %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - level=logging.DEBUG, -) - - -# Use different bucket names for async tests to avoid conflicts -BUCKET_NAME = f"{BASE_BUCKET_NAME}-aio" -OTHER_BUCKET_NAME = f"{BASE_OTHER_BUCKET_NAME}-aio" -MISSING_BUCKET_NAME = f"{BASE_MISSING_BUCKET_NAME}-aio" -INVALID_BUCKET_NAME = ".." - - -@pytest.mark.asyncio -async def test_create_bucket_aio(s3_clients_aio): - boto_client, async_light_client = s3_clients_aio - with pytest.raises(ClientError): - await boto_client.head_bucket(Bucket=OTHER_BUCKET_NAME) - await async_light_client.create_bucket(Bucket=OTHER_BUCKET_NAME) - await boto_client.head_bucket(Bucket=OTHER_BUCKET_NAME) - - -@pytest.mark.asyncio -async def test_head_bucket_exists_aio(s3_clients_aio): - """Test that head_bucket succeeds for an existing bucket.""" - _boto_client, async_light_client = s3_clients_aio - response = await async_light_client.head_bucket(Bucket=BUCKET_NAME) - - # Verify response structure - assert "BucketRegion" in response - assert "ResponseMetadata" in response - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - - -@pytest.mark.asyncio -async def test_head_bucket_not_found_aio(s3_clients_aio): - """Test that head_bucket raises NoSuchBucketError for non-existent bucket.""" - _boto_client, async_light_client = s3_clients_aio - with pytest.raises(NoSuchBucketError): - await async_light_client.head_bucket(Bucket=MISSING_BUCKET_NAME) - - -@pytest.mark.asyncio -async def test_head_object_exists_aio(s3_clients_aio): - """Test that head_object succeeds for an existing object.""" - boto_client, async_light_client = s3_clients_aio - - # First, create an object - file_content = b"test content for head_object" - key = "test_file.txt" - await boto_client.put_object( - Body=file_content, Bucket=BUCKET_NAME, Key=key, ContentType="text/plain" - ) - - # Now test head_object - response = await async_light_client.head_object(Bucket=BUCKET_NAME, Key=key) - - # Verify response structure - assert "ContentLength" in response - assert "ETag" in response - assert "LastModified" in response - assert "ResponseMetadata" in response - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - assert response["ContentLength"] == len(file_content) - assert response["ContentType"] == "text/plain" - - -@pytest.mark.asyncio -async def test_head_object_not_found_aio(s3_clients_aio): - """Test that head_object raises PresignError for non-existent object.""" - _boto_client, async_light_client = s3_clients_aio - from signurlarity.exceptions import PresignError - - with pytest.raises(PresignError): - await async_light_client.head_object( - Bucket=BUCKET_NAME, Key="nonexistent_key.txt" - ) - - -@pytest.mark.asyncio -async def test_head_object_missing_key_param_aio(s3_clients_aio): - """Test that head_object raises PresignError when Key parameter is missing.""" - _boto_client, async_light_client = s3_clients_aio - from signurlarity.exceptions import PresignError - - with pytest.raises(PresignError): - await async_light_client.head_object(Bucket=BUCKET_NAME, Key="") - - -@pytest.mark.asyncio -async def test_head_object_missing_bucket_param_aio(s3_clients_aio): - """Test that head_object raises PresignError when Bucket parameter is missing.""" - _boto_client, async_light_client = s3_clients_aio - from signurlarity.exceptions import PresignError - - with pytest.raises(PresignError): - await async_light_client.head_object(Bucket="", Key="some_key.txt") - - -@pytest.mark.asyncio -async def test_generate_presigned_post_aio(s3_clients_aio): - """Upload files using post presigned URLs with async client. - - Get a pre-signed URL with our async client, upload with httpx - check it exists with boto. - """ - boto_client, async_light_client = s3_clients_aio - - file_content, checksum = random_file(128) - key = f"{checksum}.dat" - size = len(file_content) - - fields = { - "x-amz-checksum-algorithm": CHECKSUM_ALGORITHM, - f"x-amz-checksum-{CHECKSUM_ALGORITHM}": b16_to_b64(checksum), - } - conditions = [["content-length-range", size, size]] + [ - {k: v} for k, v in fields.items() - ] - - upload_info = await async_light_client.generate_presigned_post( - Bucket=BUCKET_NAME, - Key=key, - Fields=fields, - Conditions=conditions, - ExpiresIn=60, - ) - - with httpx.Client() as client: - r = client.post( - upload_info["url"], - data=upload_info["fields"], - files={"file": file_content}, - ) - - assert r.status_code in (200, 204), r.text - await boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - - -@pytest.mark.asyncio -async def test_generate_presigned_url_aio(s3_clients_aio, caplog): - """Get a pre-signed URL with our async client, upload with httpx, check it exists with boto.""" - caplog.set_level(logging.DEBUG, logger="httpx") - caplog.set_level(logging.DEBUG, logger="httpcore") - - boto_client, async_light_client = s3_clients_aio - - file_content, checksum = random_file(128) - key = f"{checksum}.dat" - - response = await boto_client.put_object( - Body=file_content, Bucket=BUCKET_NAME, Key=key, Metadata={"Checksum": checksum} - ) - - presigned_url = await async_light_client.generate_presigned_url( - ClientMethod="get_object", - Params={"Bucket": BUCKET_NAME, "Key": key}, - ExpiresIn=3600, - ) - - with tempfile.TemporaryFile(mode="w+b") as fh: - with httpx.Client() as http_client: - response = http_client.get(presigned_url) - response.raise_for_status() - for chunk in response.iter_bytes(): - fh.write(chunk) - - -@pytest.mark.asyncio -async def test_put_object_aio(s3_clients_aio): - """Test that put_object uploads bytes to a bucket (async).""" - boto_client, async_light_client = s3_clients_aio - - file_content = b"hello from put_object async" - key = "put-object-test-aio.txt" - - response = await async_light_client.put_object( - Bucket=BUCKET_NAME, - Key=key, - Body=file_content, - ContentType="text/plain", - ) - - assert "ETag" in response - assert "ResponseMetadata" in response - assert response["ResponseMetadata"]["HTTPStatusCode"] in (200, 201) - - # Verify via boto - head = await boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - assert head["ContentLength"] == len(file_content) - - -@pytest.mark.asyncio -async def test_put_object_with_metadata_aio(s3_clients_aio): - """Test that put_object stores metadata on the object (async).""" - boto_client, async_light_client = s3_clients_aio - - file_content = b"data with metadata async" - key = "put-object-meta-test-aio.txt" - - await async_light_client.put_object( - Bucket=BUCKET_NAME, - Key=key, - Body=file_content, - Metadata={"author": "test", "version": "1"}, - ) - - head = await boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - assert head["Metadata"].get("author") == "test" - assert head["Metadata"].get("version") == "1" - - -@pytest.mark.asyncio -async def test_put_object_missing_bucket_aio(s3_clients_aio): - """Test that put_object raises PresignError when Bucket is missing (async).""" - from signurlarity.exceptions import PresignError - - _boto_client, async_light_client = s3_clients_aio - with pytest.raises(PresignError): - await async_light_client.put_object(Bucket="", Key="key.txt", Body=b"data") - - -@pytest.mark.asyncio -async def test_put_object_missing_key_aio(s3_clients_aio): - """Test that put_object raises PresignError when Key is missing (async).""" - from signurlarity.exceptions import PresignError - - _boto_client, async_light_client = s3_clients_aio - with pytest.raises(PresignError): - await async_light_client.put_object(Bucket=BUCKET_NAME, Key="", Body=b"data") - - -@pytest.mark.asyncio -async def test_list_objects_empty_aio(s3_clients_aio): - """Test list_objects on a bucket with no matching prefix (async).""" - _boto_client, async_light_client = s3_clients_aio - - response = await async_light_client.list_objects( - Bucket=BUCKET_NAME, Prefix="list-objects-nonexistent-prefix/" - ) - - assert "ResponseMetadata" in response - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - assert response["Contents"] == [] - assert response["IsTruncated"] is False - - -@pytest.mark.asyncio -async def test_list_objects_aio(s3_clients_aio): - """Test list_objects returns uploaded objects (async).""" - boto_client, async_light_client = s3_clients_aio - - keys = ["list-test-aio/a.txt", "list-test-aio/b.txt", "list-test-aio/c.txt"] - for key in keys: - await boto_client.put_object(Body=b"data", Bucket=BUCKET_NAME, Key=key) - - response = await async_light_client.list_objects( - Bucket=BUCKET_NAME, Prefix="list-test-aio/" - ) - - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - returned_keys = {obj["Key"] for obj in response["Contents"]} - assert returned_keys.issuperset(set(keys)) - for obj in response["Contents"]: - assert "Key" in obj - assert "ETag" in obj - assert "Size" in obj - assert "LastModified" in obj - - -@pytest.mark.asyncio -async def test_list_objects_with_delimiter_aio(s3_clients_aio): - """Test list_objects with delimiter groups common prefixes (async).""" - boto_client, async_light_client = s3_clients_aio - - keys = ["delim-test-aio/dir1/file.txt", "delim-test-aio/dir2/file.txt"] - for key in keys: - await boto_client.put_object(Body=b"data", Bucket=BUCKET_NAME, Key=key) - - response = await async_light_client.list_objects( - Bucket=BUCKET_NAME, Prefix="delim-test-aio/", Delimiter="/" - ) - - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - assert "CommonPrefixes" in response - prefixes = {cp["Prefix"] for cp in response["CommonPrefixes"]} - assert "delim-test-aio/dir1/" in prefixes - assert "delim-test-aio/dir2/" in prefixes - - -@pytest.mark.asyncio -async def test_list_objects_missing_bucket_aio(s3_clients_aio): - """Test that list_objects raises PresignError when Bucket is missing (async).""" - from signurlarity.exceptions import PresignError - - _boto_client, async_light_client = s3_clients_aio - with pytest.raises(PresignError): - await async_light_client.list_objects(Bucket="") - - -@pytest.mark.asyncio -async def test_copy_object_aio(s3_clients_aio): - """Test that copy_object copies an object using a string CopySource (async).""" - boto_client, async_light_client = s3_clients_aio - - src_key = "copy-src-aio.txt" - dst_key = "copy-dst-aio.txt" - await boto_client.put_object(Body=b"copy me async", Bucket=BUCKET_NAME, Key=src_key) - - response = await async_light_client.copy_object( - Bucket=BUCKET_NAME, - Key=dst_key, - CopySource=f"{BUCKET_NAME}/{src_key}", - ) - - assert "CopyObjectResult" in response - assert "ETag" in response["CopyObjectResult"] - assert "ResponseMetadata" in response - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - - head = await boto_client.head_object(Bucket=BUCKET_NAME, Key=dst_key) - assert head["ContentLength"] == len(b"copy me async") - - -@pytest.mark.asyncio -async def test_copy_object_dict_source_aio(s3_clients_aio): - """Test that copy_object works with a dict CopySource (async).""" - boto_client, async_light_client = s3_clients_aio - - src_key = "copy-src-dict-aio.txt" - dst_key = "copy-dst-dict-aio.txt" - await boto_client.put_object( - Body=b"dict source async", Bucket=BUCKET_NAME, Key=src_key - ) - - response = await async_light_client.copy_object( - Bucket=BUCKET_NAME, - Key=dst_key, - CopySource={"Bucket": BUCKET_NAME, "Key": src_key}, - ) - - assert "CopyObjectResult" in response - await boto_client.head_object(Bucket=BUCKET_NAME, Key=dst_key) - - -@pytest.mark.asyncio -async def test_copy_object_missing_bucket_aio(s3_clients_aio): - """Test that copy_object raises PresignError when Bucket is missing (async).""" - from signurlarity.exceptions import PresignError - - _boto_client, async_light_client = s3_clients_aio - with pytest.raises(PresignError): - await async_light_client.copy_object( - Bucket="", Key="dst.txt", CopySource=f"{BUCKET_NAME}/src.txt" - ) - - -@pytest.mark.asyncio -async def test_copy_object_missing_copy_source_aio(s3_clients_aio): - """Test that copy_object raises PresignError when CopySource is missing (async).""" - from signurlarity.exceptions import PresignError - - _boto_client, async_light_client = s3_clients_aio - with pytest.raises(PresignError): - await async_light_client.copy_object( - Bucket=BUCKET_NAME, Key="dst.txt", CopySource="" - ) - - -@pytest.mark.asyncio -async def test_upload_file_aio(s3_clients_aio, tmp_path): - """Test that upload_file uploads a local file to S3 (async).""" - boto_client, async_light_client = s3_clients_aio - - content = b"async file content to upload" - local_file = tmp_path / "upload_test_aio.txt" - local_file.write_bytes(content) - - key = "upload-file-test-aio.txt" - result = await async_light_client.upload_file( - Filename=str(local_file), - Bucket=BUCKET_NAME, - Key=key, - ) - - assert result is None - - head = await boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - assert head["ContentLength"] == len(content) - - -@pytest.mark.asyncio -async def test_upload_file_with_extra_args_aio(s3_clients_aio, tmp_path): - """Test that upload_file forwards ExtraArgs to put_object (async).""" - boto_client, async_light_client = s3_clients_aio - - content = b"async pdf content" - local_file = tmp_path / "report_aio.pdf" - local_file.write_bytes(content) - - key = "upload-file-extra-args-aio.pdf" - await async_light_client.upload_file( - Filename=str(local_file), - Bucket=BUCKET_NAME, - Key=key, - ExtraArgs={"ContentType": "application/pdf"}, - ) - - head = await boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - assert head["ContentType"] == "application/pdf" - assert head["ContentLength"] == len(content) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "s3_clients_aio", - [pytest.param("moto_server", marks=pytest.mark.moto)], - indirect=True, -) -async def test_upload_file_with_acl_extra_args_aio(s3_clients_aio, tmp_path): - """Test that upload_file applies ACL from ExtraArgs (moto only).""" - boto_client, async_light_client = s3_clients_aio - - content = b"async acl content" - local_file = tmp_path / "acl_aio.txt" - local_file.write_bytes(content) - - key = "upload-file-acl-aio.txt" - await async_light_client.upload_file( - Filename=str(local_file), - Bucket=BUCKET_NAME, - Key=key, - ExtraArgs={"ACL": "public-read"}, - ) - - acl = await boto_client.get_object_acl(Bucket=BUCKET_NAME, Key=key) - grants = acl.get("Grants", []) - assert any( - grant.get("Permission") == "READ" - and grant.get("Grantee", {}).get("URI") - == "http://acs.amazonaws.com/groups/global/AllUsers" - for grant in grants - ) - - -@pytest.mark.asyncio -async def test_upload_file_missing_file_aio(s3_clients_aio): - """Test that upload_file raises OSError for a non-existent file (async).""" - _boto_client, async_light_client = s3_clients_aio - with pytest.raises(OSError): - await async_light_client.upload_file( - Filename="/nonexistent/path/file.txt", - Bucket=BUCKET_NAME, - Key="key.txt", - ) - - -@pytest.mark.asyncio -async def test_delete_objects_aio(s3_clients_aio): - """Test that delete_objects deletes multiple objects (async).""" - boto_client, async_light_client = s3_clients_aio - - # Create some objects using boto - keys = ["delete-test-1.txt", "delete-test-2.txt", "delete-test-3.txt"] - for key in keys: - await boto_client.put_object(Body=b"test content", Bucket=BUCKET_NAME, Key=key) - - # Verify objects exist - for key in keys: - await boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - - # Delete objects using our async client - response = await async_light_client.delete_objects( - Bucket=BUCKET_NAME, - Delete={"Objects": [{"Key": k} for k in keys]}, - ) - - # Verify response structure - assert "ResponseMetadata" in response - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - assert "Deleted" in response - deleted_keys = {d["Key"] for d in response["Deleted"]} - assert deleted_keys == set(keys) - - # Verify objects are actually gone - for key in keys: - with pytest.raises(ClientError): - await boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - - -@pytest.mark.asyncio -async def test_delete_objects_quiet_aio(s3_clients_aio): - """Test that delete_objects with Quiet=True returns no Deleted list (async).""" - boto_client, async_light_client = s3_clients_aio - - # Create objects - keys = ["delete-quiet-1.txt", "delete-quiet-2.txt"] - for key in keys: - await boto_client.put_object(Body=b"test content", Bucket=BUCKET_NAME, Key=key) - - # Delete with Quiet=True - response = await async_light_client.delete_objects( - Bucket=BUCKET_NAME, - Delete={"Objects": [{"Key": k} for k in keys], "Quiet": True}, - ) - - assert "ResponseMetadata" in response - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - # In quiet mode, only errors are reported - assert "Errors" not in response - - # Verify objects are actually gone - for key in keys: - with pytest.raises(ClientError): - await boto_client.head_object(Bucket=BUCKET_NAME, Key=key) - - -@pytest.mark.asyncio -async def test_delete_objects_missing_bucket_aio(s3_clients_aio): - """Test that delete_objects raises PresignError for missing Bucket (async).""" - _boto_client, async_light_client = s3_clients_aio - from signurlarity.exceptions import PresignError - - with pytest.raises(PresignError): - await async_light_client.delete_objects( - Bucket="", - Delete={"Objects": [{"Key": "test.txt"}]}, - ) - - -@pytest.mark.asyncio -async def test_delete_objects_missing_objects_aio(s3_clients_aio): - """Test that delete_objects raises PresignError for missing Objects (async).""" - _boto_client, async_light_client = s3_clients_aio - from signurlarity.exceptions import PresignError - - with pytest.raises(PresignError): - await async_light_client.delete_objects(Bucket=BUCKET_NAME, Delete={}) - - with pytest.raises(PresignError): - await async_light_client.delete_objects( - Bucket=BUCKET_NAME, - Delete={"Objects": []}, - ) diff --git a/tests/test_clients.py b/tests/test_clients.py new file mode 100644 index 0000000..71bc8db --- /dev/null +++ b/tests/test_clients.py @@ -0,0 +1,875 @@ +from __future__ import annotations + +import logging +import tempfile + +import botocore +import botocore.errorfactory +import httpx +import pytest + +from signurlarity.exceptions import NoSuchBucketError, PresignError + +from .conftest import ( + BUCKET_NAME as BASE_BUCKET_NAME, +) +from .conftest import ( + CHECKSUM_ALGORITHM, + b16_to_b64, + random_file, +) +from .conftest import ( + MISSING_BUCKET_NAME as BASE_MISSING_BUCKET_NAME, +) +from .conftest import ( + OTHER_BUCKET_NAME as BASE_OTHER_BUCKET_NAME, +) + +logging.basicConfig( + format="%(levelname)s [%(asctime)s] %(name)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.DEBUG, +) + +# Bucket names for sync tests +BUCKET_NAME = f"{BASE_BUCKET_NAME}" +OTHER_BUCKET_NAME = f"{BASE_OTHER_BUCKET_NAME}" +MISSING_BUCKET_NAME = f"{BASE_MISSING_BUCKET_NAME}" + +# Bucket names for async tests (different to avoid conflicts) +BUCKET_NAME_AIO = f"{BASE_BUCKET_NAME}-aio" +OTHER_BUCKET_NAME_AIO = f"{BASE_OTHER_BUCKET_NAME}-aio" +MISSING_BUCKET_NAME_AIO = f"{BASE_MISSING_BUCKET_NAME}-aio" + + +# ============================================================================= +# SYNC TESTS +# ============================================================================= + + +def test_create_bucket(s3_clients): + boto_client, light_client = s3_clients + with pytest.raises(botocore.exceptions.ClientError): + boto_client.head_bucket(Bucket=OTHER_BUCKET_NAME) + light_client.create_bucket(Bucket=OTHER_BUCKET_NAME) + boto_client.head_bucket(Bucket=OTHER_BUCKET_NAME) + + +def test_head_bucket_exists(s3_clients): + """Test that head_bucket succeeds for an existing bucket.""" + _boto_client, light_client = s3_clients + response = light_client.head_bucket(Bucket=BUCKET_NAME) + assert "BucketRegion" in response + assert "ResponseMetadata" in response + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + +def test_head_bucket_not_found(s3_clients): + """Test that head_bucket raises NoSuchBucketError for non-existent bucket.""" + _boto_client, light_client = s3_clients + with pytest.raises(NoSuchBucketError): + light_client.head_bucket(Bucket=MISSING_BUCKET_NAME) + + +def test_head_object_exists(s3_clients): + """Test that head_object succeeds for an existing object.""" + boto_client, light_client = s3_clients + file_content = b"test content for head_object" + key = "test_file.txt" + boto_client.put_object( + Body=file_content, Bucket=BUCKET_NAME, Key=key, ContentType="text/plain" + ) + response = light_client.head_object(Bucket=BUCKET_NAME, Key=key) + assert "ContentLength" in response + assert "ETag" in response + assert "LastModified" in response + assert "ResponseMetadata" in response + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert response["ContentLength"] == len(file_content) + assert response["ContentType"] == "text/plain" + + +def test_head_object_not_found(s3_clients): + """Test that head_object raises PresignError for non-existent object.""" + _boto_client, light_client = s3_clients + with pytest.raises(PresignError): + light_client.head_object(Bucket=BUCKET_NAME, Key="nonexistent_key.txt") + + +def test_head_object_missing_key_param(s3_clients): + """Test that head_object raises PresignError when Key parameter is missing.""" + _boto_client, light_client = s3_clients + with pytest.raises(PresignError): + light_client.head_object(Bucket=BUCKET_NAME, Key="") + + +def test_head_object_missing_bucket_param(s3_clients): + """Test that head_object raises PresignError when Bucket parameter is missing.""" + _boto_client, light_client = s3_clients + with pytest.raises(PresignError): + light_client.head_object(Bucket="", Key="some_key.txt") + + +def test_generate_presigned_post(s3_clients): + """Upload files using post presigned URLs.""" + boto_client, light_client = s3_clients + file_content, checksum = random_file(128) + key = f"{checksum}.dat" + size = len(file_content) + fields = { + "x-amz-checksum-algorithm": CHECKSUM_ALGORITHM, + f"x-amz-checksum-{CHECKSUM_ALGORITHM}": b16_to_b64(checksum), + } + conditions = [["content-length-range", size, size]] + [ + {k: v} for k, v in fields.items() + ] + upload_info = light_client.generate_presigned_post( + Bucket=BUCKET_NAME, Key=key, Fields=fields, Conditions=conditions, ExpiresIn=60 + ) + with httpx.Client() as client: + r = client.post( + upload_info["url"], data=upload_info["fields"], files={"file": file_content} + ) + assert r.status_code in (200, 204), r.text + boto_client.head_object(Bucket=BUCKET_NAME, Key=key) + + +def test_generate_presigned_url(s3_clients, caplog): + """Get a pre-signed URL with our client, upload with httpx, check it exists with boto.""" + caplog.set_level(logging.DEBUG, logger="httpx") + caplog.set_level(logging.DEBUG, logger="httpcore") + boto_client, light_client = s3_clients + file_content, checksum = random_file(128) + key = f"{checksum}.dat" + response = boto_client.put_object( + Body=file_content, Bucket=BUCKET_NAME, Key=key, Metadata={"Checksum": checksum} + ) + presigned_url = light_client.generate_presigned_url( + ClientMethod="get_object", + Params={"Bucket": BUCKET_NAME, "Key": key}, + ExpiresIn=3600, + ) + with tempfile.TemporaryFile(mode="w+b") as fh: + with httpx.Client() as http_client: + response = http_client.get(presigned_url) + response.raise_for_status() + for chunk in response.iter_bytes(): + fh.write(chunk) + + +def test_put_object(s3_clients): + """Test that put_object uploads bytes to a bucket.""" + boto_client, light_client = s3_clients + file_content = b"hello from put_object" + key = "put-object-test.txt" + response = light_client.put_object( + Bucket=BUCKET_NAME, Key=key, Body=file_content, ContentType="text/plain" + ) + assert "ETag" in response + assert "ResponseMetadata" in response + assert response["ResponseMetadata"]["HTTPStatusCode"] in (200, 201) + head = boto_client.head_object(Bucket=BUCKET_NAME, Key=key) + assert head["ContentLength"] == len(file_content) + + +def test_put_object_with_metadata(s3_clients): + """Test that put_object stores metadata on the object.""" + boto_client, light_client = s3_clients + file_content = b"data with metadata" + key = "put-object-meta-test.txt" + light_client.put_object( + Bucket=BUCKET_NAME, + Key=key, + Body=file_content, + Metadata={"author": "test", "version": "1"}, + ) + head = boto_client.head_object(Bucket=BUCKET_NAME, Key=key) + assert head["Metadata"].get("author") == "test" + assert head["Metadata"].get("version") == "1" + + +def test_put_object_missing_bucket(s3_clients): + """Test that put_object raises PresignError when Bucket is missing.""" + _boto_client, light_client = s3_clients + with pytest.raises(PresignError): + light_client.put_object(Bucket="", Key="key.txt", Body=b"data") + + +def test_put_object_missing_key(s3_clients): + """Test that put_object raises PresignError when Key is missing.""" + _boto_client, light_client = s3_clients + with pytest.raises(PresignError): + light_client.put_object(Bucket=BUCKET_NAME, Key="", Body=b"data") + + +def test_list_objects_empty(s3_clients): + """Test list_objects on a bucket with no matching prefix.""" + _boto_client, light_client = s3_clients + response = light_client.list_objects( + Bucket=BUCKET_NAME, Prefix="list-objects-nonexistent-prefix/" + ) + assert "ResponseMetadata" in response + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert response["Contents"] == [] + assert response["IsTruncated"] is False + + +def test_list_objects(s3_clients): + """Test list_objects returns uploaded objects.""" + boto_client, light_client = s3_clients + keys = ["list-test/a.txt", "list-test/b.txt", "list-test/c.txt"] + for key in keys: + boto_client.put_object(Body=b"data", Bucket=BUCKET_NAME, Key=key) + response = light_client.list_objects(Bucket=BUCKET_NAME, Prefix="list-test/") + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + returned_keys = {obj["Key"] for obj in response["Contents"]} + assert returned_keys.issuperset(set(keys)) + for obj in response["Contents"]: + assert "Key" in obj + assert "ETag" in obj + assert "Size" in obj + assert "LastModified" in obj + + +def test_list_objects_with_delimiter(s3_clients): + """Test list_objects with delimiter groups common prefixes.""" + boto_client, light_client = s3_clients + keys = ["delim-test/dir1/file.txt", "delim-test/dir2/file.txt"] + for key in keys: + boto_client.put_object(Body=b"data", Bucket=BUCKET_NAME, Key=key) + response = light_client.list_objects( + Bucket=BUCKET_NAME, Prefix="delim-test/", Delimiter="/" + ) + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert "CommonPrefixes" in response + prefixes = {cp["Prefix"] for cp in response["CommonPrefixes"]} + assert "delim-test/dir1/" in prefixes + assert "delim-test/dir2/" in prefixes + + +def test_list_objects_missing_bucket(s3_clients): + """Test that list_objects raises PresignError when Bucket is missing.""" + _boto_client, light_client = s3_clients + with pytest.raises(PresignError): + light_client.list_objects(Bucket="") + + +def test_copy_object(s3_clients): + """Test that copy_object copies an object using a string CopySource.""" + boto_client, light_client = s3_clients + src_key = "copy-src.txt" + dst_key = "copy-dst.txt" + boto_client.put_object(Body=b"copy me", Bucket=BUCKET_NAME, Key=src_key) + response = light_client.copy_object( + Bucket=BUCKET_NAME, Key=dst_key, CopySource=f"{BUCKET_NAME}/{src_key}" + ) + assert "CopyObjectResult" in response + assert "ETag" in response["CopyObjectResult"] + assert "ResponseMetadata" in response + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + head = boto_client.head_object(Bucket=BUCKET_NAME, Key=dst_key) + assert head["ContentLength"] == len(b"copy me") + + +def test_copy_object_dict_source(s3_clients): + """Test that copy_object works with a dict CopySource.""" + boto_client, light_client = s3_clients + src_key = "copy-src-dict.txt" + dst_key = "copy-dst-dict.txt" + boto_client.put_object(Body=b"dict source", Bucket=BUCKET_NAME, Key=src_key) + response = light_client.copy_object( + Bucket=BUCKET_NAME, + Key=dst_key, + CopySource={"Bucket": BUCKET_NAME, "Key": src_key}, + ) + assert "CopyObjectResult" in response + boto_client.head_object(Bucket=BUCKET_NAME, Key=dst_key) + + +def test_copy_object_missing_bucket(s3_clients): + """Test that copy_object raises PresignError when Bucket is missing.""" + _boto_client, light_client = s3_clients + with pytest.raises(PresignError): + light_client.copy_object( + Bucket="", Key="dst.txt", CopySource=f"{BUCKET_NAME}/src.txt" + ) + + +def test_copy_object_missing_copy_source(s3_clients): + """Test that copy_object raises PresignError when CopySource is missing.""" + _boto_client, light_client = s3_clients + with pytest.raises(PresignError): + light_client.copy_object(Bucket=BUCKET_NAME, Key="dst.txt", CopySource="") + + +def test_upload_file(s3_clients, tmp_path): + """Test that upload_file uploads a local file to S3.""" + boto_client, light_client = s3_clients + content = b"file content to upload" + local_file = tmp_path / "upload_test.txt" + local_file.write_bytes(content) + key = "upload-file-test.txt" + result = light_client.upload_file( + Filename=str(local_file), Bucket=BUCKET_NAME, Key=key + ) + assert result is None + head = boto_client.head_object(Bucket=BUCKET_NAME, Key=key) + assert head["ContentLength"] == len(content) + + +def test_upload_file_with_extra_args(s3_clients, tmp_path): + """Test that upload_file forwards ExtraArgs to put_object.""" + boto_client, light_client = s3_clients + content = b"pdf content" + local_file = tmp_path / "report.pdf" + local_file.write_bytes(content) + key = "upload-file-extra-args.pdf" + light_client.upload_file( + Filename=str(local_file), + Bucket=BUCKET_NAME, + Key=key, + ExtraArgs={"ContentType": "application/pdf"}, + ) + head = boto_client.head_object(Bucket=BUCKET_NAME, Key=key) + assert head["ContentType"] == "application/pdf" + assert head["ContentLength"] == len(content) + + +@pytest.mark.parametrize( + "s3_clients", + [pytest.param("moto_server", marks=pytest.mark.moto)], + indirect=True, +) +def test_upload_file_with_acl_extra_args(s3_clients, tmp_path): + """Test that upload_file applies ACL from ExtraArgs (moto only).""" + boto_client, light_client = s3_clients + content = b"acl content" + local_file = tmp_path / "acl.txt" + local_file.write_bytes(content) + key = "upload-file-acl.txt" + light_client.upload_file( + Filename=str(local_file), + Bucket=BUCKET_NAME, + Key=key, + ExtraArgs={"ACL": "public-read"}, + ) + acl = boto_client.get_object_acl(Bucket=BUCKET_NAME, Key=key) + grants = acl.get("Grants", []) + assert any( + grant.get("Permission") == "READ" + and grant.get("Grantee", {}).get("URI") + == "http://acs.amazonaws.com/groups/global/AllUsers" + for grant in grants + ), grants + + +def test_upload_file_missing_file(s3_clients): + """Test that upload_file raises OSError for a non-existent file.""" + _boto_client, light_client = s3_clients + with pytest.raises(OSError): + light_client.upload_file( + Filename="/nonexistent/path/file.txt", Bucket=BUCKET_NAME, Key="key.txt" + ) + + +def test_delete_objects(s3_clients): + """Test that delete_objects deletes multiple objects.""" + boto_client, light_client = s3_clients + keys = ["delete-test-1.txt", "delete-test-2.txt", "delete-test-3.txt"] + for key in keys: + boto_client.put_object(Body=b"test content", Bucket=BUCKET_NAME, Key=key) + for key in keys: + boto_client.head_object(Bucket=BUCKET_NAME, Key=key) + response = light_client.delete_objects( + Bucket=BUCKET_NAME, Delete={"Objects": [{"Key": k} for k in keys]} + ) + assert "ResponseMetadata" in response + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert "Deleted" in response + deleted_keys = {d["Key"] for d in response["Deleted"]} + assert deleted_keys == set(keys) + for key in keys: + with pytest.raises(botocore.exceptions.ClientError): + boto_client.head_object(Bucket=BUCKET_NAME, Key=key) + + +def test_delete_objects_quiet(s3_clients): + """Test that delete_objects with Quiet=True returns no Deleted list.""" + boto_client, light_client = s3_clients + keys = ["delete-quiet-1.txt", "delete-quiet-2.txt"] + for key in keys: + boto_client.put_object(Body=b"test content", Bucket=BUCKET_NAME, Key=key) + response = light_client.delete_objects( + Bucket=BUCKET_NAME, + Delete={"Objects": [{"Key": k} for k in keys], "Quiet": True}, + ) + assert "ResponseMetadata" in response + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert "Errors" not in response + for key in keys: + with pytest.raises(botocore.exceptions.ClientError): + boto_client.head_object(Bucket=BUCKET_NAME, Key=key) + + +def test_delete_objects_missing_bucket(s3_clients): + """Test that delete_objects raises PresignError for missing Bucket.""" + _boto_client, light_client = s3_clients + with pytest.raises(PresignError): + light_client.delete_objects( + Bucket="", Delete={"Objects": [{"Key": "test.txt"}]} + ) + + +def test_delete_objects_missing_objects(s3_clients): + """Test that delete_objects raises PresignError for missing Objects.""" + _boto_client, light_client = s3_clients + with pytest.raises(PresignError): + light_client.delete_objects(Bucket=BUCKET_NAME, Delete={}) + with pytest.raises(PresignError): + light_client.delete_objects(Bucket=BUCKET_NAME, Delete={"Objects": []}) + + +# ============================================================================= +# ASYNC TESTS +# ============================================================================= + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "s3_clients_aio", + [pytest.param("moto_server", marks=pytest.mark.moto)], + indirect=True, +) +async def test_create_bucket_aio(s3_clients_aio): + boto_client, async_light_client = s3_clients_aio + with pytest.raises(botocore.errorfactory.ClientError): + await boto_client.head_bucket(Bucket=OTHER_BUCKET_NAME_AIO) + await async_light_client.create_bucket(Bucket=OTHER_BUCKET_NAME_AIO) + await boto_client.head_bucket(Bucket=OTHER_BUCKET_NAME_AIO) + + +@pytest.mark.asyncio +async def test_head_bucket_exists_aio(s3_clients_aio): + """Test that head_bucket succeeds for an existing bucket (async).""" + _boto_client, async_light_client = s3_clients_aio + response = await async_light_client.head_bucket(Bucket=BUCKET_NAME_AIO) + assert "BucketRegion" in response + assert "ResponseMetadata" in response + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + +@pytest.mark.asyncio +async def test_head_bucket_not_found_aio(s3_clients_aio): + """Test that head_bucket raises NoSuchBucketError for non-existent bucket (async).""" + _boto_client, async_light_client = s3_clients_aio + with pytest.raises(NoSuchBucketError): + await async_light_client.head_bucket(Bucket=MISSING_BUCKET_NAME_AIO) + + +@pytest.mark.asyncio +async def test_head_object_exists_aio(s3_clients_aio): + """Test that head_object succeeds for an existing object (async).""" + boto_client, async_light_client = s3_clients_aio + file_content = b"test content for head_object" + key = "test_file_aio.txt" + await boto_client.put_object( + Body=file_content, Bucket=BUCKET_NAME_AIO, Key=key, ContentType="text/plain" + ) + response = await async_light_client.head_object(Bucket=BUCKET_NAME_AIO, Key=key) + assert "ContentLength" in response + assert "ETag" in response + assert "LastModified" in response + assert "ResponseMetadata" in response + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert response["ContentLength"] == len(file_content) + assert response["ContentType"] == "text/plain" + + +@pytest.mark.asyncio +async def test_head_object_not_found_aio(s3_clients_aio): + """Test that head_object raises PresignError for non-existent object (async).""" + _boto_client, async_light_client = s3_clients_aio + with pytest.raises(PresignError): + await async_light_client.head_object( + Bucket=BUCKET_NAME_AIO, Key="nonexistent_key.txt" + ) + + +@pytest.mark.asyncio +async def test_head_object_missing_key_param_aio(s3_clients_aio): + """Test that head_object raises PresignError when Key parameter is missing (async).""" + _boto_client, async_light_client = s3_clients_aio + with pytest.raises(PresignError): + await async_light_client.head_object(Bucket=BUCKET_NAME_AIO, Key="") + + +@pytest.mark.asyncio +async def test_head_object_missing_bucket_param_aio(s3_clients_aio): + """Test that head_object raises PresignError when Bucket parameter is missing (async).""" + _boto_client, async_light_client = s3_clients_aio + with pytest.raises(PresignError): + await async_light_client.head_object(Bucket="", Key="some_key.txt") + + +@pytest.mark.asyncio +async def test_generate_presigned_post_aio(s3_clients_aio): + """Upload files using post presigned URLs with async client.""" + boto_client, async_light_client = s3_clients_aio + file_content, checksum = random_file(128) + key = f"{checksum}_aio.dat" + size = len(file_content) + fields = { + "x-amz-checksum-algorithm": CHECKSUM_ALGORITHM, + f"x-amz-checksum-{CHECKSUM_ALGORITHM}": b16_to_b64(checksum), + } + conditions = [["content-length-range", size, size]] + [ + {k: v} for k, v in fields.items() + ] + upload_info = await async_light_client.generate_presigned_post( + Bucket=BUCKET_NAME_AIO, + Key=key, + Fields=fields, + Conditions=conditions, + ExpiresIn=60, + ) + with httpx.Client() as client: + r = client.post( + upload_info["url"], data=upload_info["fields"], files={"file": file_content} + ) + assert r.status_code in (200, 204), r.text + await boto_client.head_object(Bucket=BUCKET_NAME_AIO, Key=key) + + +@pytest.mark.asyncio +async def test_generate_presigned_url_aio(s3_clients_aio, caplog): + """Get a pre-signed URL with our async client, upload with httpx, check it exists with boto.""" + caplog.set_level(logging.DEBUG, logger="httpx") + caplog.set_level(logging.DEBUG, logger="httpcore") + boto_client, async_light_client = s3_clients_aio + file_content, checksum = random_file(128) + key = f"{checksum}_aio.dat" + response = await boto_client.put_object( + Body=file_content, + Bucket=BUCKET_NAME_AIO, + Key=key, + Metadata={"Checksum": checksum}, + ) + presigned_url = await async_light_client.generate_presigned_url( + ClientMethod="get_object", + Params={"Bucket": BUCKET_NAME_AIO, "Key": key}, + ExpiresIn=3600, + ) + with tempfile.TemporaryFile(mode="w+b") as fh: + with httpx.Client() as http_client: + response = http_client.get(presigned_url) + response.raise_for_status() + for chunk in response.iter_bytes(): + fh.write(chunk) + + +@pytest.mark.asyncio +async def test_put_object_aio(s3_clients_aio): + """Test that put_object uploads bytes to a bucket (async).""" + boto_client, async_light_client = s3_clients_aio + file_content = b"hello from put_object async" + key = "put-object-test-aio.txt" + response = await async_light_client.put_object( + Bucket=BUCKET_NAME_AIO, Key=key, Body=file_content, ContentType="text/plain" + ) + assert "ETag" in response + assert "ResponseMetadata" in response + assert response["ResponseMetadata"]["HTTPStatusCode"] in (200, 201) + head = await boto_client.head_object(Bucket=BUCKET_NAME_AIO, Key=key) + assert head["ContentLength"] == len(file_content) + + +@pytest.mark.asyncio +async def test_put_object_with_metadata_aio(s3_clients_aio): + """Test that put_object stores metadata on the object (async).""" + boto_client, async_light_client = s3_clients_aio + file_content = b"data with metadata async" + key = "put-object-meta-test-aio.txt" + await async_light_client.put_object( + Bucket=BUCKET_NAME_AIO, + Key=key, + Body=file_content, + Metadata={"author": "test", "version": "1"}, + ) + head = await boto_client.head_object(Bucket=BUCKET_NAME_AIO, Key=key) + assert head["Metadata"].get("author") == "test" + assert head["Metadata"].get("version") == "1" + + +@pytest.mark.asyncio +async def test_put_object_missing_bucket_aio(s3_clients_aio): + """Test that put_object raises PresignError when Bucket is missing (async).""" + _boto_client, async_light_client = s3_clients_aio + with pytest.raises(PresignError): + await async_light_client.put_object(Bucket="", Key="key.txt", Body=b"data") + + +@pytest.mark.asyncio +async def test_put_object_missing_key_aio(s3_clients_aio): + """Test that put_object raises PresignError when Key is missing (async).""" + _boto_client, async_light_client = s3_clients_aio + with pytest.raises(PresignError): + await async_light_client.put_object( + Bucket=BUCKET_NAME_AIO, Key="", Body=b"data" + ) + + +@pytest.mark.asyncio +async def test_list_objects_empty_aio(s3_clients_aio): + """Test list_objects on a bucket with no matching prefix (async).""" + _boto_client, async_light_client = s3_clients_aio + response = await async_light_client.list_objects( + Bucket=BUCKET_NAME_AIO, Prefix="list-objects-nonexistent-prefix/" + ) + assert "ResponseMetadata" in response + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert response["Contents"] == [] + assert response["IsTruncated"] is False + + +@pytest.mark.asyncio +async def test_list_objects_aio(s3_clients_aio): + """Test list_objects returns uploaded objects (async).""" + boto_client, async_light_client = s3_clients_aio + keys = ["list-test-aio/a.txt", "list-test-aio/b.txt", "list-test-aio/c.txt"] + for key in keys: + await boto_client.put_object(Body=b"data", Bucket=BUCKET_NAME_AIO, Key=key) + response = await async_light_client.list_objects( + Bucket=BUCKET_NAME_AIO, Prefix="list-test-aio/" + ) + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + returned_keys = {obj["Key"] for obj in response["Contents"]} + assert returned_keys.issuperset(set(keys)) + for obj in response["Contents"]: + assert "Key" in obj + assert "ETag" in obj + assert "Size" in obj + assert "LastModified" in obj + + +@pytest.mark.asyncio +async def test_list_objects_with_delimiter_aio(s3_clients_aio): + """Test list_objects with delimiter groups common prefixes (async).""" + boto_client, async_light_client = s3_clients_aio + keys = ["delim-test-aio/dir1/file.txt", "delim-test-aio/dir2/file.txt"] + for key in keys: + await boto_client.put_object(Body=b"data", Bucket=BUCKET_NAME_AIO, Key=key) + response = await async_light_client.list_objects( + Bucket=BUCKET_NAME_AIO, Prefix="delim-test-aio/", Delimiter="/" + ) + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert "CommonPrefixes" in response + prefixes = {cp["Prefix"] for cp in response["CommonPrefixes"]} + assert "delim-test-aio/dir1/" in prefixes + assert "delim-test-aio/dir2/" in prefixes + + +@pytest.mark.asyncio +async def test_list_objects_missing_bucket_aio(s3_clients_aio): + """Test that list_objects raises PresignError when Bucket is missing (async).""" + _boto_client, async_light_client = s3_clients_aio + with pytest.raises(PresignError): + await async_light_client.list_objects(Bucket="") + + +@pytest.mark.asyncio +async def test_copy_object_aio(s3_clients_aio): + """Test that copy_object copies an object using a string CopySource (async).""" + boto_client, async_light_client = s3_clients_aio + src_key = "copy-src-aio.txt" + dst_key = "copy-dst-aio.txt" + await boto_client.put_object( + Body=b"copy me async", Bucket=BUCKET_NAME_AIO, Key=src_key + ) + response = await async_light_client.copy_object( + Bucket=BUCKET_NAME_AIO, Key=dst_key, CopySource=f"{BUCKET_NAME_AIO}/{src_key}" + ) + assert "CopyObjectResult" in response + assert "ETag" in response["CopyObjectResult"] + assert "ResponseMetadata" in response + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + head = await boto_client.head_object(Bucket=BUCKET_NAME_AIO, Key=dst_key) + assert head["ContentLength"] == len(b"copy me async") + + +@pytest.mark.asyncio +async def test_copy_object_dict_source_aio(s3_clients_aio): + """Test that copy_object works with a dict CopySource (async).""" + boto_client, async_light_client = s3_clients_aio + src_key = "copy-src-dict-aio.txt" + dst_key = "copy-dst-dict-aio.txt" + await boto_client.put_object( + Body=b"dict source async", Bucket=BUCKET_NAME_AIO, Key=src_key + ) + response = await async_light_client.copy_object( + Bucket=BUCKET_NAME_AIO, + Key=dst_key, + CopySource={"Bucket": BUCKET_NAME_AIO, "Key": src_key}, + ) + assert "CopyObjectResult" in response + await boto_client.head_object(Bucket=BUCKET_NAME_AIO, Key=dst_key) + + +@pytest.mark.asyncio +async def test_copy_object_missing_bucket_aio(s3_clients_aio): + """Test that copy_object raises PresignError when Bucket is missing (async).""" + _boto_client, async_light_client = s3_clients_aio + with pytest.raises(PresignError): + await async_light_client.copy_object( + Bucket="", Key="dst.txt", CopySource=f"{BUCKET_NAME_AIO}/src.txt" + ) + + +@pytest.mark.asyncio +async def test_copy_object_missing_copy_source_aio(s3_clients_aio): + """Test that copy_object raises PresignError when CopySource is missing (async).""" + _boto_client, async_light_client = s3_clients_aio + with pytest.raises(PresignError): + await async_light_client.copy_object( + Bucket=BUCKET_NAME_AIO, Key="dst.txt", CopySource="" + ) + + +@pytest.mark.asyncio +async def test_upload_file_aio(s3_clients_aio, tmp_path): + """Test that upload_file uploads a local file to S3 (async).""" + boto_client, async_light_client = s3_clients_aio + content = b"async file content to upload" + local_file = tmp_path / "upload_test_aio.txt" + local_file.write_bytes(content) + key = "upload-file-test-aio.txt" + result = await async_light_client.upload_file( + Filename=str(local_file), Bucket=BUCKET_NAME_AIO, Key=key + ) + assert result is None + head = await boto_client.head_object(Bucket=BUCKET_NAME_AIO, Key=key) + assert head["ContentLength"] == len(content) + + +@pytest.mark.asyncio +async def test_upload_file_with_extra_args_aio(s3_clients_aio, tmp_path): + """Test that upload_file forwards ExtraArgs to put_object (async).""" + boto_client, async_light_client = s3_clients_aio + content = b"async pdf content" + local_file = tmp_path / "report_aio.pdf" + local_file.write_bytes(content) + key = "upload-file-extra-args-aio.pdf" + await async_light_client.upload_file( + Filename=str(local_file), + Bucket=BUCKET_NAME_AIO, + Key=key, + ExtraArgs={"ContentType": "application/pdf"}, + ) + head = await boto_client.head_object(Bucket=BUCKET_NAME_AIO, Key=key) + assert head["ContentType"] == "application/pdf" + assert head["ContentLength"] == len(content) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "s3_clients_aio", + [pytest.param("moto_server", marks=pytest.mark.moto)], + indirect=True, +) +async def test_upload_file_with_acl_extra_args_aio(s3_clients_aio, tmp_path): + """Test that upload_file applies ACL from ExtraArgs (moto only, async).""" + boto_client, async_light_client = s3_clients_aio + content = b"async acl content" + local_file = tmp_path / "acl_aio.txt" + local_file.write_bytes(content) + key = "upload-file-acl-aio.txt" + await async_light_client.upload_file( + Filename=str(local_file), + Bucket=BUCKET_NAME_AIO, + Key=key, + ExtraArgs={"ACL": "public-read"}, + ) + acl = await boto_client.get_object_acl(Bucket=BUCKET_NAME_AIO, Key=key) + grants = acl.get("Grants", []) + assert any( + grant.get("Permission") == "READ" + and grant.get("Grantee", {}).get("URI") + == "http://acs.amazonaws.com/groups/global/AllUsers" + for grant in grants + ) + + +@pytest.mark.asyncio +async def test_upload_file_missing_file_aio(s3_clients_aio): + """Test that upload_file raises OSError for a non-existent file (async).""" + _boto_client, async_light_client = s3_clients_aio + with pytest.raises(OSError): + await async_light_client.upload_file( + Filename="/nonexistent/path/file.txt", Bucket=BUCKET_NAME_AIO, Key="key.txt" + ) + + +@pytest.mark.asyncio +async def test_delete_objects_aio(s3_clients_aio): + """Test that delete_objects deletes multiple objects (async).""" + boto_client, async_light_client = s3_clients_aio + keys = ["delete-test-1.txt", "delete-test-2.txt", "delete-test-3.txt"] + for key in keys: + await boto_client.put_object( + Body=b"test content", Bucket=BUCKET_NAME_AIO, Key=key + ) + for key in keys: + await boto_client.head_object(Bucket=BUCKET_NAME_AIO, Key=key) + response = await async_light_client.delete_objects( + Bucket=BUCKET_NAME_AIO, Delete={"Objects": [{"Key": k} for k in keys]} + ) + assert "ResponseMetadata" in response + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert "Deleted" in response + deleted_keys = {d["Key"] for d in response["Deleted"]} + assert deleted_keys == set(keys) + for key in keys: + with pytest.raises(botocore.errorfactory.ClientError): + await boto_client.head_object(Bucket=BUCKET_NAME_AIO, Key=key) + + +@pytest.mark.asyncio +async def test_delete_objects_quiet_aio(s3_clients_aio): + """Test that delete_objects with Quiet=True returns no Deleted list (async).""" + boto_client, async_light_client = s3_clients_aio + keys = ["delete-quiet-1.txt", "delete-quiet-2.txt"] + for key in keys: + await boto_client.put_object( + Body=b"test content", Bucket=BUCKET_NAME_AIO, Key=key + ) + response = await async_light_client.delete_objects( + Bucket=BUCKET_NAME_AIO, + Delete={"Objects": [{"Key": k} for k in keys], "Quiet": True}, + ) + assert "ResponseMetadata" in response + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert "Errors" not in response + for key in keys: + with pytest.raises(botocore.errorfactory.ClientError): + await boto_client.head_object(Bucket=BUCKET_NAME_AIO, Key=key) + + +@pytest.mark.asyncio +async def test_delete_objects_missing_bucket_aio(s3_clients_aio): + """Test that delete_objects raises PresignError for missing Bucket (async).""" + _boto_client, async_light_client = s3_clients_aio + with pytest.raises(PresignError): + await async_light_client.delete_objects( + Bucket="", Delete={"Objects": [{"Key": "test.txt"}]} + ) + + +@pytest.mark.asyncio +async def test_delete_objects_missing_objects_aio(s3_clients_aio): + """Test that delete_objects raises PresignError for missing Objects (async).""" + _boto_client, async_light_client = s3_clients_aio + with pytest.raises(PresignError): + await async_light_client.delete_objects(Bucket=BUCKET_NAME_AIO, Delete={}) + with pytest.raises(PresignError): + await async_light_client.delete_objects( + Bucket=BUCKET_NAME_AIO, Delete={"Objects": []} + )