Release V0.1.2 #3
@@ -634,7 +634,15 @@ class ObjectStorage:
|
||||
if part_number < 1:
|
||||
raise StorageError("part_number must be >= 1")
|
||||
bucket_path = self._bucket_path(bucket_name)
|
||||
manifest, upload_root = self._load_multipart_manifest(bucket_path.name, upload_id)
|
||||
|
||||
# Get the upload root directory
|
||||
upload_root = self._multipart_dir(bucket_path.name, upload_id)
|
||||
if not upload_root.exists():
|
||||
upload_root = self._legacy_multipart_dir(bucket_path.name, upload_id)
|
||||
if not upload_root.exists():
|
||||
raise StorageError("Multipart upload not found")
|
||||
|
||||
# Write the part data first (can happen concurrently)
|
||||
checksum = hashlib.md5()
|
||||
part_filename = f"part-{part_number:05d}.part"
|
||||
part_path = upload_root / part_filename
|
||||
@@ -645,9 +653,23 @@ class ObjectStorage:
|
||||
"size": part_path.stat().st_size,
|
||||
"filename": part_filename,
|
||||
}
|
||||
|
||||
# Update manifest with file locking to prevent race conditions
|
||||
manifest_path = upload_root / self.MULTIPART_MANIFEST
|
||||
lock_path = upload_root / ".manifest.lock"
|
||||
|
||||
with lock_path.open("w") as lock_file:
|
||||
with _file_lock(lock_file):
|
||||
# Re-read manifest under lock to get latest state
|
||||
try:
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise StorageError("Multipart manifest unreadable") from exc
|
||||
|
||||
parts = manifest.setdefault("parts", {})
|
||||
parts[str(part_number)] = record
|
||||
self._write_multipart_manifest(upload_root, manifest)
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
|
||||
return record["etag"]
|
||||
|
||||
def complete_multipart_upload(
|
||||
|
||||
28
tests/test_boto3_multipart.py
Normal file
28
tests/test_boto3_multipart.py
Normal file
@@ -0,0 +1,28 @@
|
||||
import uuid
|
||||
import pytest
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_boto3_multipart_upload(live_server):
|
||||
bucket_name = f'mp-test-{uuid.uuid4().hex[:8]}'
|
||||
object_key = 'large-file.bin'
|
||||
s3 = boto3.client('s3', endpoint_url=live_server, aws_access_key_id='test', aws_secret_access_key='secret', region_name='us-east-1', use_ssl=False, config=Config(signature_version='s3v4', retries={'max_attempts': 1}, s3={'addressing_style': 'path'}))
|
||||
s3.create_bucket(Bucket=bucket_name)
|
||||
try:
|
||||
response = s3.create_multipart_upload(Bucket=bucket_name, Key=object_key)
|
||||
upload_id = response['UploadId']
|
||||
parts = []
|
||||
part1_data = b'A' * 1024
|
||||
part2_data = b'B' * 1024
|
||||
resp1 = s3.upload_part(Bucket=bucket_name, Key=object_key, PartNumber=1, UploadId=upload_id, Body=part1_data)
|
||||
parts.append({'PartNumber': 1, 'ETag': resp1['ETag']})
|
||||
resp2 = s3.upload_part(Bucket=bucket_name, Key=object_key, PartNumber=2, UploadId=upload_id, Body=part2_data)
|
||||
parts.append({'PartNumber': 2, 'ETag': resp2['ETag']})
|
||||
s3.complete_multipart_upload(Bucket=bucket_name, Key=object_key, UploadId=upload_id, MultipartUpload={'Parts': parts})
|
||||
obj = s3.get_object(Bucket=bucket_name, Key=object_key)
|
||||
content = obj['Body'].read()
|
||||
assert content == part1_data + part2_data
|
||||
s3.delete_object(Bucket=bucket_name, Key=object_key)
|
||||
finally:
|
||||
s3.delete_bucket(Bucket=bucket_name)
|
||||
Reference in New Issue
Block a user