kqjy/MyFSIO
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

Fix Remove fallback ETag, make etag optional, fix multipart ETag storage, fix request entity too large error due to mishandled multipart uploads

Browse Source
This commit is contained in:
kqjy
2026-01-01 21:51:01 +08:00
parent e792b86485
commit b3dce8d13e
5 changed files with 417 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
app/s3_api.py
View File

@@ -1314,7 +1314,8 @@ def _bucket_list_versions_handler(bucket_name: str) -> Response:
SubElement(version, "VersionId").text = "null"
SubElement(version, "IsLatest").text = "true"
SubElement(version, "LastModified").text = obj.last_modified.strftime("%Y-%m-%dT%H:%M:%S.000Z")
SubElement(version, "ETag").text = f'"{obj.etag}"'
if obj.etag:
SubElement(version, "ETag").text = f'"{obj.etag}"'
SubElement(version, "Size").text = str(obj.size)
SubElement(version, "StorageClass").text = "STANDARD"
@@ -2178,10 +2179,11 @@ def bucket_handler(bucket_name: str) -> Response:
obj_el = SubElement(root, "Contents")
SubElement(obj_el, "Key").text = meta.key
SubElement(obj_el, "LastModified").text = meta.last_modified.isoformat()
SubElement(obj_el, "ETag").text = f'"{meta.etag}"'
if meta.etag:
SubElement(obj_el, "ETag").text = f'"{meta.etag}"'
SubElement(obj_el, "Size").text = str(meta.size)
SubElement(obj_el, "StorageClass").text = "STANDARD"
for cp in common_prefixes:
cp_el = SubElement(root, "CommonPrefixes")
SubElement(cp_el, "Prefix").text = cp
@@ -2194,15 +2196,16 @@ def bucket_handler(bucket_name: str) -> Response:
SubElement(root, "IsTruncated").text = "true" if is_truncated else "false"
if delimiter:
SubElement(root, "Delimiter").text = delimiter
if is_truncated and delimiter and next_marker:
SubElement(root, "NextMarker").text = next_marker
for meta in objects:
obj_el = SubElement(root, "Contents")
SubElement(obj_el, "Key").text = meta.key
SubElement(obj_el, "LastModified").text = meta.last_modified.isoformat()
SubElement(obj_el, "ETag").text = f'"{meta.etag}"'
if meta.etag:
SubElement(obj_el, "ETag").text = f'"{meta.etag}"'
SubElement(obj_el, "Size").text = str(meta.size)
for cp in common_prefixes:
@@ -2282,7 +2285,8 @@ def object_handler(bucket_name: str, object_key: str):
extra={"bucket": bucket_name, "key": object_key, "size": meta.size},
)
response = Response(status=200)
response.headers["ETag"] = f'"{meta.etag}"'
if meta.etag:
response.headers["ETag"] = f'"{meta.etag}"'
_notifications().emit_object_created(
bucket_name,
@@ -2725,7 +2729,8 @@ def _copy_object(dest_bucket: str, dest_key: str, copy_source: str) -> Response:
root = Element("CopyObjectResult")
SubElement(root, "LastModified").text = meta.last_modified.isoformat()
SubElement(root, "ETag").text = f'"{meta.etag}"'
if meta.etag:
SubElement(root, "ETag").text = f'"{meta.etag}"'
return _xml_response(root)
@@ -2947,8 +2952,9 @@ def _complete_multipart_upload(bucket_name: str, object_key: str) -> Response:
SubElement(root, "Location").text = location
SubElement(root, "Bucket").text = bucket_name
SubElement(root, "Key").text = object_key
SubElement(root, "ETag").text = f'"{meta.etag}"'
if meta.etag:
SubElement(root, "ETag").text = f'"{meta.etag}"'
return _xml_response(root)

22
app/storage.py
View File

@@ -90,7 +90,7 @@ class ObjectMeta:
key: str
size: int
last_modified: datetime
etag: str
etag: Optional[str] = None
metadata: Optional[Dict[str, str]] = None
@@ -1079,11 +1079,6 @@ class ObjectStorage:
checksum.update(data)
target.write(data)
metadata = manifest.get("metadata")
if metadata:
self._write_metadata(bucket_id, safe_key, metadata)
else:
self._delete_metadata(bucket_id, safe_key)
except BlockingIOError:
raise StorageError("Another upload to this key is in progress")
finally:
@@ -1097,12 +1092,18 @@ class ObjectStorage:
self._invalidate_bucket_stats_cache(bucket_id)
stat = destination.stat()
# Performance: Lazy update - only update the affected key instead of invalidating whole cache
etag = checksum.hexdigest()
metadata = manifest.get("metadata")
internal_meta = {"__etag__": etag, "__size__": str(stat.st_size)}
combined_meta = {**internal_meta, **(metadata or {})}
self._write_metadata(bucket_id, safe_key, combined_meta)
obj_meta = ObjectMeta(
key=safe_key.as_posix(),
size=stat.st_size,
last_modified=datetime.fromtimestamp(stat.st_mtime, timezone.utc),
etag=checksum.hexdigest(),
etag=etag,
metadata=metadata,
)
self._update_object_cache_entry(bucket_id, safe_key.as_posix(), obj_meta)
@@ -1369,10 +1370,7 @@ class ObjectStorage:
stat = entry.stat()
etag = meta_cache.get(key)
if not etag:
etag = f'"{stat.st_size}-{int(stat.st_mtime)}"'
objects[key] = ObjectMeta(
key=key,
size=stat.st_size,

12
app/ui.py
View File

@@ -563,6 +563,7 @@ def initiate_multipart_upload(bucket_name: str):
@ui_bp.put("/buckets/<bucket_name>/multipart/<upload_id>/parts")
@limiter.exempt
def upload_multipart_part(bucket_name: str, upload_id: str):
principal = _current_principal()
try:
@@ -606,9 +607,14 @@ def complete_multipart_upload(bucket_name: str, upload_id: str):
normalized.append({"part_number": number, "etag": etag})
try:
result = _storage().complete_multipart_upload(bucket_name, upload_id, normalized)
_replication().trigger_replication(bucket_name, result["key"])
return jsonify(result)
_replication().trigger_replication(bucket_name, result.key)
return jsonify({
"key": result.key,
"size": result.size,
"etag": result.etag,
"last_modified": result.last_modified.isoformat() if result.last_modified else None,
})
except StorageError as exc:
return jsonify({"error": str(exc)}), 400