Release V0.1.2 #3

Merged
kqjy merged 10 commits from next into main 2025-11-26 04:59:15 +00:00
2 changed files with 54 additions and 4 deletions
Showing only changes of commit aaa230b19b - Show all commits

View File

@@ -634,7 +634,15 @@ class ObjectStorage:
if part_number < 1:
raise StorageError("part_number must be >= 1")
bucket_path = self._bucket_path(bucket_name)
manifest, upload_root = self._load_multipart_manifest(bucket_path.name, upload_id)
# Get the upload root directory
upload_root = self._multipart_dir(bucket_path.name, upload_id)
if not upload_root.exists():
upload_root = self._legacy_multipart_dir(bucket_path.name, upload_id)
if not upload_root.exists():
raise StorageError("Multipart upload not found")
# Write the part data first (can happen concurrently)
checksum = hashlib.md5()
part_filename = f"part-{part_number:05d}.part"
part_path = upload_root / part_filename
@@ -645,9 +653,23 @@ class ObjectStorage:
"size": part_path.stat().st_size,
"filename": part_filename,
}
parts = manifest.setdefault("parts", {})
parts[str(part_number)] = record
self._write_multipart_manifest(upload_root, manifest)
# Update manifest with file locking to prevent race conditions
manifest_path = upload_root / self.MULTIPART_MANIFEST
lock_path = upload_root / ".manifest.lock"
with lock_path.open("w") as lock_file:
with _file_lock(lock_file):
# Re-read manifest under lock to get latest state
try:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise StorageError("Multipart manifest unreadable") from exc
parts = manifest.setdefault("parts", {})
parts[str(part_number)] = record
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
return record["etag"]
def complete_multipart_upload(

View File

@@ -0,0 +1,28 @@
import uuid
import pytest
import boto3
from botocore.client import Config
@pytest.mark.integration
def test_boto3_multipart_upload(live_server):
bucket_name = f'mp-test-{uuid.uuid4().hex[:8]}'
object_key = 'large-file.bin'
s3 = boto3.client('s3', endpoint_url=live_server, aws_access_key_id='test', aws_secret_access_key='secret', region_name='us-east-1', use_ssl=False, config=Config(signature_version='s3v4', retries={'max_attempts': 1}, s3={'addressing_style': 'path'}))
s3.create_bucket(Bucket=bucket_name)
try:
response = s3.create_multipart_upload(Bucket=bucket_name, Key=object_key)
upload_id = response['UploadId']
parts = []
part1_data = b'A' * 1024
part2_data = b'B' * 1024
resp1 = s3.upload_part(Bucket=bucket_name, Key=object_key, PartNumber=1, UploadId=upload_id, Body=part1_data)
parts.append({'PartNumber': 1, 'ETag': resp1['ETag']})
resp2 = s3.upload_part(Bucket=bucket_name, Key=object_key, PartNumber=2, UploadId=upload_id, Body=part2_data)
parts.append({'PartNumber': 2, 'ETag': resp2['ETag']})
s3.complete_multipart_upload(Bucket=bucket_name, Key=object_key, UploadId=upload_id, MultipartUpload={'Parts': parts})
obj = s3.get_object(Bucket=bucket_name, Key=object_key)
content = obj['Body'].read()
assert content == part1_data + part2_data
s3.delete_object(Bucket=bucket_name, Key=object_key)
finally:
s3.delete_bucket(Bucket=bucket_name)