kqjy/MyFSIO
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

Debug replication corruption issue - check if it's boto3 issue

Browse Source
This commit is contained in:
kqjy
2025-12-23 12:02:26 +08:00
parent 6e659902bd
commit 90893cac27
1 changed files with 16 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
app/replication.py
View File

@@ -317,7 +317,13 @@ class ReplicationManager:
read_timeout=REPLICATION_READ_TIMEOUT, read_timeout=REPLICATION_READ_TIMEOUT,
retries={'max_attempts': 2}, # Limited retries to prevent long hangs retries={'max_attempts': 2}, # Limited retries to prevent long hangs
signature_version='s3v4', # Force signature v4 for compatibility signature_version='s3v4', # Force signature v4 for compatibility
s3={'addressing_style': 'path'} # Use path-style addressing for compatibility s3={
'addressing_style': 'path', # Use path-style addressing for compatibility
},
# Disable SDK automatic checksums - they cause SignatureDoesNotMatch errors
# with S3-compatible servers that don't support CRC32 checksum headers
request_checksum_calculation='when_required',
response_checksum_validation='when_required',
) )
s3 = boto3.client( s3 = boto3.client(
"s3", "s3",
@@ -347,7 +353,11 @@ class ReplicationManager:
extra_args = {} extra_args = {}
if metadata: if metadata:
extra_args["Metadata"] = metadata # Filter out metadata with None or empty values to prevent signature issues
# boto3 signs all metadata headers, so None values cause mismatches
filtered_metadata = {k: v for k, v in metadata.items() if v is not None and v != ''}
if filtered_metadata:
extra_args["Metadata"] = filtered_metadata
# Guess content type to prevent corruption/wrong handling # Guess content type to prevent corruption/wrong handling
content_type, _ = mimetypes.guess_type(path) content_type, _ = mimetypes.guess_type(path)
@@ -371,8 +381,11 @@ class ReplicationManager:
} }
if content_type: if content_type:
put_kwargs["ContentType"] = content_type put_kwargs["ContentType"] = content_type
# Only add metadata if we have valid filtered values
if metadata: if metadata:
put_kwargs["Metadata"] = metadata filtered = {k: v for k, v in metadata.items() if v is not None and v != ''}
if filtered:
put_kwargs["Metadata"] = filtered
# Debug logging for signature issues # Debug logging for signature issues
logger.debug(f"PUT request details: bucket={rule.target_bucket}, key={repr(object_key)}, " logger.debug(f"PUT request details: bucket={rule.target_bucket}, key={repr(object_key)}, "