Fix rclone CopyObject SignatureDoesNotMatch caused by internal metadata leaking as X-Amz-Meta headers

app/s3_api.py

@@ -305,16 +305,10 @@ def _verify_sigv4_header(req: Any, auth_header: str) -> Principal | None:
             header_values, payload_hash, amz_date, date_stamp, region,
             service, secret_key, signature,
         ):
-            if current_app.config.get("DEBUG_SIGV4"):
-                logger.warning("SigV4 signature mismatch for %s %s", req.method, req.path)
             raise IamError("SignatureDoesNotMatch")
     else:
         method = req.method
-        query_args = []
+        query_args = sorted(req.args.items(multi=True), key=lambda x: (x[0], x[1]))
-        for key, value in req.args.items(multi=True):
-            query_args.append((key, value))
-        query_args.sort(key=lambda x: (x[0], x[1]))
-
         canonical_query_parts = []
         for k, v in query_args:
             canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}")
@@ -339,8 +333,6 @@ def _verify_sigv4_header(req: Any, auth_header: str) -> Principal | None:
         string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}"
         calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
         if not hmac.compare_digest(calculated_signature, signature):
-            if current_app.config.get("DEBUG_SIGV4"):
-                logger.warning("SigV4 signature mismatch for %s %s", method, req.path)
             raise IamError("SignatureDoesNotMatch")
 
     session_token = req.headers.get("X-Amz-Security-Token")
@@ -682,7 +674,7 @@ def _extract_request_metadata() -> Dict[str, str]:
     for header, value in request.headers.items():
         if header.lower().startswith("x-amz-meta-"):
             key = header[11:]
-            if key:
+            if key and not (key.startswith("__") and key.endswith("__")):
                 metadata[key] = value
     return metadata
 
@@ -1039,6 +1031,8 @@ def _apply_object_headers(
     response.headers["ETag"] = f'"{etag}"'
     response.headers["Accept-Ranges"] = "bytes"
     for key, value in (metadata or {}).items():
+        if key.startswith("__") and key.endswith("__"):
+            continue
         safe_value = _sanitize_header_value(str(value))
         response.headers[f"X-Amz-Meta-{key}"] = safe_value
 
@@ -2467,7 +2461,7 @@ def _post_object(bucket_name: str) -> Response:
     for field_name, value in request.form.items():
         if field_name.lower().startswith("x-amz-meta-"):
             key = field_name[11:]
-            if key:
+            if key and not (key.startswith("__") and key.endswith("__")):
                 metadata[key] = value
     try:
         meta = storage.put_object(bucket_name, object_key, file.stream, metadata=metadata or None)
@@ -3445,7 +3439,7 @@ def _copy_object(dest_bucket: str, dest_key: str, copy_source: str) -> Response:
         if validation_error:
             return _error_response("InvalidArgument", validation_error, 400)
     else:
-        metadata = source_metadata
+        metadata = {k: v for k, v in source_metadata.items() if not (k.startswith("__") and k.endswith("__"))}
 
     try:
         with source_path.open("rb") as stream:

app/storage.py

@@ -16,7 +16,7 @@ from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager
 from dataclasses import dataclass
 from datetime import datetime, timezone
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from typing import Any, BinaryIO, Dict, Generator, List, Optional
 
 try:
@@ -573,6 +573,10 @@ class ObjectStorage:
                         try:
                             st = entry.stat()
                             etag = meta_cache.get(key)
+                            if etag is None:
+                                safe_key = PurePosixPath(key)
+                                meta = self._read_metadata(bucket_id, Path(safe_key))
+                                etag = meta.get("__etag__") if meta else None
                             entries_files.append((key, st.st_size, st.st_mtime, etag))
                         except OSError:
                             pass
@@ -2094,16 +2098,15 @@ class ObjectStorage:
 
     def _update_etag_index(self, bucket_id: str, key: str, etag: Optional[str]) -> None:
         etag_index_path = self._system_bucket_root(bucket_id) / "etag_index.json"
+        if not etag_index_path.exists():
+            return
         try:
-            index: Dict[str, str] = {}
+            with open(etag_index_path, 'r', encoding='utf-8') as f:
-            if etag_index_path.exists():
+                index = json.load(f)
-                with open(etag_index_path, 'r', encoding='utf-8') as f:
-                    index = json.load(f)
             if etag is None:
                 index.pop(key, None)
             else:
                 index[key] = etag
-            etag_index_path.parent.mkdir(parents=True, exist_ok=True)
             with open(etag_index_path, 'w', encoding='utf-8') as f:
                 json.dump(index, f)
         except (OSError, json.JSONDecodeError):

app/version.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-APP_VERSION = "0.3.2"
+APP_VERSION = "0.3.3"
 
 
 def get_version() -> str: