MyFSIO v0.3.2 Release

Reviewed-on: #25

.gitignore

@@ -26,6 +26,10 @@ dist/
 *.egg-info/
 .eggs/
 
+# Rust / maturin build artifacts
+myfsio_core/target/
+myfsio_core/Cargo.lock
+
 # Local runtime artifacts
 logs/
 *.log

app/__init__.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import html as html_module
 import logging
 import mimetypes
+import os
 import shutil
 import sys
 import time
@@ -93,7 +94,14 @@ def create_app(
         app.config.setdefault("WTF_CSRF_ENABLED", False)
 
     # Trust X-Forwarded-* headers from proxies
-    app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1)
+    num_proxies = app.config.get("NUM_TRUSTED_PROXIES", 1)
+    if num_proxies:
+        if "NUM_TRUSTED_PROXIES" not in os.environ:
+            logging.getLogger(__name__).warning(
+                "NUM_TRUSTED_PROXIES not set, defaulting to 1. "
+                "Set NUM_TRUSTED_PROXIES=0 if not behind a reverse proxy."
+            )
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_for=num_proxies, x_proto=num_proxies, x_host=num_proxies, x_prefix=num_proxies)
 
     # Enable gzip compression for responses (10-20x smaller JSON payloads)
     if app.config.get("ENABLE_GZIP", True):
@@ -107,7 +115,7 @@ def create_app(
 
     storage = ObjectStorage(
         Path(app.config["STORAGE_ROOT"]),
-        cache_ttl=app.config.get("OBJECT_CACHE_TTL", 5),
+        cache_ttl=app.config.get("OBJECT_CACHE_TTL", 60),
         object_cache_max_size=app.config.get("OBJECT_CACHE_MAX_SIZE", 100),
         bucket_config_cache_ttl=app.config.get("BUCKET_CONFIG_CACHE_TTL_SECONDS", 30.0),
         object_key_max_length_bytes=app.config.get("OBJECT_KEY_MAX_LENGTH_BYTES", 1024),

app/bucket_policies.py

@@ -75,7 +75,7 @@ def _evaluate_condition_operator(
         expected_null = condition_values[0].lower() in ("true", "1", "yes") if condition_values else True
         return is_null == expected_null
 
-    return True
+    return False
 
 ACTION_ALIASES = {
     "s3:listbucket": "list",

app/config.py

@@ -241,7 +241,7 @@ class AppConfig:
         cors_expose_headers = _csv(str(_get("CORS_EXPOSE_HEADERS", "*")), ["*"])
         session_lifetime_days = int(_get("SESSION_LIFETIME_DAYS", 30))
         bucket_stats_cache_ttl = int(_get("BUCKET_STATS_CACHE_TTL", 60))
-        object_cache_ttl = int(_get("OBJECT_CACHE_TTL", 5))
+        object_cache_ttl = int(_get("OBJECT_CACHE_TTL", 60))
 
         encryption_enabled = str(_get("ENCRYPTION_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
         encryption_keys_dir = storage_root / ".myfsio.sys" / "keys"
@@ -314,7 +314,7 @@ class AppConfig:
         site_region = str(_get("SITE_REGION", "us-east-1"))
         site_priority = int(_get("SITE_PRIORITY", 100))
         ratelimit_admin = _validate_rate_limit(str(_get("RATE_LIMIT_ADMIN", "60 per minute")))
-        num_trusted_proxies = int(_get("NUM_TRUSTED_PROXIES", 0))
+        num_trusted_proxies = int(_get("NUM_TRUSTED_PROXIES", 1))
         allowed_redirect_hosts_raw = _get("ALLOWED_REDIRECT_HOSTS", "")
         allowed_redirect_hosts = [h.strip() for h in str(allowed_redirect_hosts_raw).split(",") if h.strip()]
         allow_internal_endpoints = str(_get("ALLOW_INTERNAL_ENDPOINTS", "0")).lower() in {"1", "true", "yes", "on"}

app/encrypted_storage.py

@@ -189,7 +189,13 @@ class EncryptedObjectStorage:
     
     def list_objects(self, bucket_name: str, **kwargs):
         return self.storage.list_objects(bucket_name, **kwargs)
-    
+
+    def list_objects_shallow(self, bucket_name: str, **kwargs):
+        return self.storage.list_objects_shallow(bucket_name, **kwargs)
+
+    def search_objects(self, bucket_name: str, query: str, **kwargs):
+        return self.storage.search_objects(bucket_name, query, **kwargs)
+
     def list_objects_all(self, bucket_name: str):
         return self.storage.list_objects_all(bucket_name)
     

app/iam.py

@@ -164,9 +164,14 @@ class IamService:
         self._clear_failed_attempts(access_key)
         return self._build_principal(access_key, record)
 
+    _MAX_LOCKOUT_KEYS = 10000
+
     def _record_failed_attempt(self, access_key: str) -> None:
         if not access_key:
             return
+        if access_key not in self._failed_attempts and len(self._failed_attempts) >= self._MAX_LOCKOUT_KEYS:
+            oldest_key = min(self._failed_attempts, key=lambda k: self._failed_attempts[k][0] if self._failed_attempts[k] else datetime.min.replace(tzinfo=timezone.utc))
+            del self._failed_attempts[oldest_key]
         attempts = self._failed_attempts.setdefault(access_key, deque())
         self._prune_attempts(attempts)
         attempts.append(datetime.now(timezone.utc))

app/notifications.py

@@ -15,29 +15,23 @@ from typing import Any, Dict, List, Optional
 from urllib.parse import urlparse
 
 import requests
+from urllib3.util.connection import create_connection as _urllib3_create_connection
 
 
-def _is_safe_url(url: str, allow_internal: bool = False) -> bool:
-    """Check if a URL is safe to make requests to (not internal/private).
-
-    Args:
-        url: The URL to check.
-        allow_internal: If True, allows internal/private IP addresses.
-                       Use for self-hosted deployments on internal networks.
-    """
+def _resolve_and_check_url(url: str, allow_internal: bool = False) -> Optional[str]:
     try:
         parsed = urlparse(url)
         hostname = parsed.hostname
         if not hostname:
-            return False
+            return None
         cloud_metadata_hosts = {
             "metadata.google.internal",
             "169.254.169.254",
         }
         if hostname.lower() in cloud_metadata_hosts:
-            return False
+            return None
         if allow_internal:
-            return True
+            return hostname
         blocked_hosts = {
             "localhost",
             "127.0.0.1",
@@ -46,17 +40,46 @@ def _is_safe_url(url: str, allow_internal: bool = False) -> bool:
             "[::1]",
         }
         if hostname.lower() in blocked_hosts:
-            return False
+            return None
         try:
             resolved_ip = socket.gethostbyname(hostname)
             ip = ipaddress.ip_address(resolved_ip)
             if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
-                return False
+                return None
+            return resolved_ip
         except (socket.gaierror, ValueError):
-            return False
-        return True
+            return None
     except Exception:
-        return False
+        return None
+
+
+def _is_safe_url(url: str, allow_internal: bool = False) -> bool:
+    return _resolve_and_check_url(url, allow_internal) is not None
+
+
+_dns_pin_lock = threading.Lock()
+
+
+def _pinned_post(url: str, pinned_ip: str, **kwargs: Any) -> requests.Response:
+    parsed = urlparse(url)
+    hostname = parsed.hostname or ""
+    session = requests.Session()
+    original_create = _urllib3_create_connection
+
+    def _create_pinned(address: Any, *args: Any, **kw: Any) -> Any:
+        host, req_port = address
+        if host == hostname:
+            return original_create((pinned_ip, req_port), *args, **kw)
+        return original_create(address, *args, **kw)
+
+    import urllib3.util.connection as _conn_mod
+    with _dns_pin_lock:
+        _conn_mod.create_connection = _create_pinned
+        try:
+            return session.post(url, **kwargs)
+        finally:
+            _conn_mod.create_connection = original_create
+
 
 logger = logging.getLogger(__name__)
 
@@ -344,16 +367,18 @@ class NotificationService:
                 self._queue.task_done()
 
     def _send_notification(self, event: NotificationEvent, destination: WebhookDestination) -> None:
-        if not _is_safe_url(destination.url, allow_internal=self._allow_internal_endpoints):
-            raise RuntimeError(f"Blocked request to cloud metadata service (SSRF protection): {destination.url}")
+        resolved_ip = _resolve_and_check_url(destination.url, allow_internal=self._allow_internal_endpoints)
+        if not resolved_ip:
+            raise RuntimeError(f"Blocked request (SSRF protection): {destination.url}")
         payload = event.to_s3_event()
         headers = {"Content-Type": "application/json", **destination.headers}
 
         last_error = None
         for attempt in range(destination.retry_count):
             try:
-                response = requests.post(
+                response = _pinned_post(
                     destination.url,
+                    resolved_ip,
                     json=payload,
                     headers=headers,
                     timeout=destination.timeout_seconds,

app/s3_api.py

@@ -267,39 +267,6 @@ def _verify_sigv4_header(req: Any, auth_header: str) -> Principal | None:
     if not secret_key:
         raise IamError("SignatureDoesNotMatch")
 
-    method = req.method
-    canonical_uri = _get_canonical_uri(req)
-    
-    query_args = []
-    for key, value in req.args.items(multi=True):
-        query_args.append((key, value))
-    query_args.sort(key=lambda x: (x[0], x[1]))
-    
-    canonical_query_parts = []
-    for k, v in query_args:
-        canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}")
-    canonical_query_string = "&".join(canonical_query_parts)
-
-    signed_headers_list = signed_headers_str.split(";")
-    canonical_headers_parts = []
-    for header in signed_headers_list:
-        header_val = req.headers.get(header)
-        if header_val is None:
-             header_val = ""
-        
-        if header.lower() == 'expect' and header_val == "":
-            header_val = "100-continue"
-        
-        header_val = " ".join(header_val.split())
-        canonical_headers_parts.append(f"{header.lower()}:{header_val}\n")
-    canonical_headers = "".join(canonical_headers_parts)
-
-    payload_hash = req.headers.get("X-Amz-Content-Sha256")
-    if not payload_hash:
-        payload_hash = hashlib.sha256(req.get_data()).hexdigest()
-
-    canonical_request = f"{method}\n{canonical_uri}\n{canonical_query_string}\n{canonical_headers}\n{signed_headers_str}\n{payload_hash}"
-
     amz_date = req.headers.get("X-Amz-Date") or req.headers.get("Date")
     if not amz_date:
         raise IamError("Missing Date header")
@@ -321,23 +288,60 @@ def _verify_sigv4_header(req: Any, auth_header: str) -> Principal | None:
         if 'date' in signed_headers_set:
             required_headers.remove('x-amz-date')
             required_headers.add('date')
-        
+
         if not required_headers.issubset(signed_headers_set):
              raise IamError("Required headers not signed")
 
-    credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
-    signing_key = _get_signature_key(secret_key, date_stamp, region, service)
+    canonical_uri = _get_canonical_uri(req)
+    payload_hash = req.headers.get("X-Amz-Content-Sha256")
+    if not payload_hash:
+        payload_hash = hashlib.sha256(req.get_data()).hexdigest()
+
     if _HAS_RUST:
-        string_to_sign = _rc.build_string_to_sign(amz_date, credential_scope, canonical_request)
-        calculated_signature = _rc.compute_signature(signing_key, string_to_sign)
+        query_params = list(req.args.items(multi=True))
+        header_values = [(h, req.headers.get(h) or "") for h in signed_headers_str.split(";")]
+        if not _rc.verify_sigv4_signature(
+            req.method, canonical_uri, query_params, signed_headers_str,
+            header_values, payload_hash, amz_date, date_stamp, region,
+            service, secret_key, signature,
+        ):
+            if current_app.config.get("DEBUG_SIGV4"):
+                logger.warning("SigV4 signature mismatch for %s %s", req.method, req.path)
+            raise IamError("SignatureDoesNotMatch")
     else:
+        method = req.method
+        query_args = []
+        for key, value in req.args.items(multi=True):
+            query_args.append((key, value))
+        query_args.sort(key=lambda x: (x[0], x[1]))
+
+        canonical_query_parts = []
+        for k, v in query_args:
+            canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}")
+        canonical_query_string = "&".join(canonical_query_parts)
+
+        signed_headers_list = signed_headers_str.split(";")
+        canonical_headers_parts = []
+        for header in signed_headers_list:
+            header_val = req.headers.get(header)
+            if header_val is None:
+                 header_val = ""
+            if header.lower() == 'expect' and header_val == "":
+                header_val = "100-continue"
+            header_val = " ".join(header_val.split())
+            canonical_headers_parts.append(f"{header.lower()}:{header_val}\n")
+        canonical_headers = "".join(canonical_headers_parts)
+
+        canonical_request = f"{method}\n{canonical_uri}\n{canonical_query_string}\n{canonical_headers}\n{signed_headers_str}\n{payload_hash}"
+
+        credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
+        signing_key = _get_signature_key(secret_key, date_stamp, region, service)
         string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}"
         calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
-
-    if not hmac.compare_digest(calculated_signature, signature):
-        if current_app.config.get("DEBUG_SIGV4"):
-            logger.warning("SigV4 signature mismatch for %s %s", method, req.path)
-        raise IamError("SignatureDoesNotMatch")
+        if not hmac.compare_digest(calculated_signature, signature):
+            if current_app.config.get("DEBUG_SIGV4"):
+                logger.warning("SigV4 signature mismatch for %s %s", method, req.path)
+            raise IamError("SignatureDoesNotMatch")
 
     session_token = req.headers.get("X-Amz-Security-Token")
     if session_token:
@@ -366,14 +370,21 @@ def _verify_sigv4_query(req: Any) -> Principal | None:
         req_time = datetime.strptime(amz_date, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc)
     except ValueError:
         raise IamError("Invalid Date format")
-    
+
     now = datetime.now(timezone.utc)
+    tolerance = timedelta(seconds=current_app.config.get("SIGV4_TIMESTAMP_TOLERANCE_SECONDS", 900))
+    if req_time > now + tolerance:
+        raise IamError("Request date is too far in the future")
     try:
         expires_seconds = int(expires)
         if expires_seconds <= 0:
             raise IamError("Invalid Expires value: must be positive")
     except ValueError:
         raise IamError("Invalid Expires value: must be an integer")
+    min_expiry = current_app.config.get("PRESIGNED_URL_MIN_EXPIRY_SECONDS", 1)
+    max_expiry = current_app.config.get("PRESIGNED_URL_MAX_EXPIRY_SECONDS", 604800)
+    if expires_seconds < min_expiry or expires_seconds > max_expiry:
+        raise IamError(f"Expiration must be between {min_expiry} second(s) and {max_expiry} seconds")
     if now > req_time + timedelta(seconds=expires_seconds):
         raise IamError("Request expired")
 
@@ -381,53 +392,58 @@ def _verify_sigv4_query(req: Any) -> Principal | None:
     if not secret_key:
         raise IamError("Invalid access key")
 
-    method = req.method
     canonical_uri = _get_canonical_uri(req)
-    
-    query_args = []
-    for key, value in req.args.items(multi=True):
-        if key != "X-Amz-Signature":
-            query_args.append((key, value))
-    query_args.sort(key=lambda x: (x[0], x[1]))
-    
-    canonical_query_parts = []
-    for k, v in query_args:
-        canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}")
-    canonical_query_string = "&".join(canonical_query_parts)
-    
-    signed_headers_list = signed_headers_str.split(";")
-    canonical_headers_parts = []
-    for header in signed_headers_list:
-        val = req.headers.get(header, "").strip()
-        if header.lower() == 'expect' and val == "":
-            val = "100-continue"
-        val = " ".join(val.split())
-        canonical_headers_parts.append(f"{header.lower()}:{val}\n")
-    canonical_headers = "".join(canonical_headers_parts)
-    
-    payload_hash = "UNSIGNED-PAYLOAD"
-    
-    canonical_request = "\n".join([
-        method,
-        canonical_uri,
-        canonical_query_string,
-        canonical_headers,
-        signed_headers_str,
-        payload_hash
-    ])
-    
-    credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
-    signing_key = _get_signature_key(secret_key, date_stamp, region, service)
+
     if _HAS_RUST:
-        string_to_sign = _rc.build_string_to_sign(amz_date, credential_scope, canonical_request)
-        calculated_signature = _rc.compute_signature(signing_key, string_to_sign)
+        query_params = [(k, v) for k, v in req.args.items(multi=True) if k != "X-Amz-Signature"]
+        header_values = [(h, req.headers.get(h) or "") for h in signed_headers_str.split(";")]
+        if not _rc.verify_sigv4_signature(
+            req.method, canonical_uri, query_params, signed_headers_str,
+            header_values, "UNSIGNED-PAYLOAD", amz_date, date_stamp, region,
+            service, secret_key, signature,
+        ):
+            raise IamError("SignatureDoesNotMatch")
     else:
+        method = req.method
+        query_args = []
+        for key, value in req.args.items(multi=True):
+            if key != "X-Amz-Signature":
+                query_args.append((key, value))
+        query_args.sort(key=lambda x: (x[0], x[1]))
+
+        canonical_query_parts = []
+        for k, v in query_args:
+            canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}")
+        canonical_query_string = "&".join(canonical_query_parts)
+
+        signed_headers_list = signed_headers_str.split(";")
+        canonical_headers_parts = []
+        for header in signed_headers_list:
+            val = req.headers.get(header, "").strip()
+            if header.lower() == 'expect' and val == "":
+                val = "100-continue"
+            val = " ".join(val.split())
+            canonical_headers_parts.append(f"{header.lower()}:{val}\n")
+        canonical_headers = "".join(canonical_headers_parts)
+
+        payload_hash = "UNSIGNED-PAYLOAD"
+
+        canonical_request = "\n".join([
+            method,
+            canonical_uri,
+            canonical_query_string,
+            canonical_headers,
+            signed_headers_str,
+            payload_hash
+        ])
+
+        credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
+        signing_key = _get_signature_key(secret_key, date_stamp, region, service)
         hashed_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
         string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashed_request}"
         calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
-
-    if not hmac.compare_digest(calculated_signature, signature):
-        raise IamError("SignatureDoesNotMatch")
+        if not hmac.compare_digest(calculated_signature, signature):
+            raise IamError("SignatureDoesNotMatch")
 
     session_token = req.args.get("X-Amz-Security-Token")
     if session_token:
@@ -586,7 +602,11 @@ def _validate_presigned_request(action: str, bucket_name: str, object_key: str)
         request_time = datetime.strptime(amz_date, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc)
     except ValueError as exc:
         raise IamError("Invalid X-Amz-Date") from exc
-    if datetime.now(timezone.utc) > request_time + timedelta(seconds=expiry):
+    now = datetime.now(timezone.utc)
+    tolerance = timedelta(seconds=current_app.config.get("SIGV4_TIMESTAMP_TOLERANCE_SECONDS", 900))
+    if request_time > now + tolerance:
+        raise IamError("Request date is too far in the future")
+    if now > request_time + timedelta(seconds=expiry):
         raise IamError("Presigned URL expired")
 
     signed_headers_list = [header.strip().lower() for header in signed_headers.split(";") if header]
@@ -986,7 +1006,7 @@ def _render_encryption_document(config: dict[str, Any]) -> Element:
     return root
 
 
-def _stream_file(path, chunk_size: int = 64 * 1024):
+def _stream_file(path, chunk_size: int = 256 * 1024):
     with path.open("rb") as handle:
         while True:
             chunk = handle.read(chunk_size)
@@ -1039,6 +1059,7 @@ def _maybe_handle_bucket_subresource(bucket_name: str) -> Response | None:
         "logging": _bucket_logging_handler,
         "uploads": _bucket_uploads_handler,
         "policy": _bucket_policy_handler,
+        "policyStatus": _bucket_policy_status_handler,
         "replication": _bucket_replication_handler,
         "website": _bucket_website_handler,
     }
@@ -1321,8 +1342,8 @@ def _bucket_cors_handler(bucket_name: str) -> Response:
 
 
 def _bucket_encryption_handler(bucket_name: str) -> Response:
-    if request.method not in {"GET", "PUT"}:
-        return _method_not_allowed(["GET", "PUT"])
+    if request.method not in {"GET", "PUT", "DELETE"}:
+        return _method_not_allowed(["GET", "PUT", "DELETE"])
     principal, error = _require_principal()
     if error:
         return error
@@ -1343,6 +1364,13 @@ def _bucket_encryption_handler(bucket_name: str) -> Response:
                 404,
             )
         return _xml_response(_render_encryption_document(config))
+    if request.method == "DELETE":
+        try:
+            storage.set_bucket_encryption(bucket_name, None)
+        except StorageError as exc:
+            return _error_response("NoSuchBucket", str(exc), 404)
+        current_app.logger.info("Bucket encryption deleted", extra={"bucket": bucket_name})
+        return Response(status=204)
     ct_error = _require_xml_content_type()
     if ct_error:
         return ct_error
@@ -1439,6 +1467,99 @@ def _bucket_acl_handler(bucket_name: str) -> Response:
     return _xml_response(root)
 
 
+def _object_acl_handler(bucket_name: str, object_key: str) -> Response:
+    from .acl import create_canned_acl, GRANTEE_ALL_USERS, GRANTEE_AUTHENTICATED_USERS
+
+    if request.method not in {"GET", "PUT"}:
+        return _method_not_allowed(["GET", "PUT"])
+    storage = _storage()
+    try:
+        path = storage.get_object_path(bucket_name, object_key)
+    except (StorageError, FileNotFoundError):
+        return _error_response("NoSuchKey", "Object not found", 404)
+
+    if request.method == "PUT":
+        principal, error = _object_principal("write", bucket_name, object_key)
+        if error:
+            return error
+        owner_id = principal.access_key if principal else "anonymous"
+        canned_acl = request.headers.get("x-amz-acl", "private")
+        acl = create_canned_acl(canned_acl, owner_id)
+        acl_service = _acl()
+        metadata = storage.get_object_metadata(bucket_name, object_key)
+        metadata.update(acl_service.create_object_acl_metadata(acl))
+        safe_key = storage._sanitize_object_key(object_key, storage._object_key_max_length_bytes)
+        storage._write_metadata(bucket_name, safe_key, metadata)
+        current_app.logger.info("Object ACL set", extra={"bucket": bucket_name, "key": object_key, "acl": canned_acl})
+        return Response(status=200)
+
+    principal, error = _object_principal("read", bucket_name, object_key)
+    if error:
+        return error
+    owner_id = principal.access_key if principal else "anonymous"
+    acl_service = _acl()
+    metadata = storage.get_object_metadata(bucket_name, object_key)
+    acl = acl_service.get_object_acl(bucket_name, object_key, metadata)
+    if not acl:
+        acl = create_canned_acl("private", owner_id)
+
+    root = Element("AccessControlPolicy")
+    owner_el = SubElement(root, "Owner")
+    SubElement(owner_el, "ID").text = acl.owner
+    SubElement(owner_el, "DisplayName").text = acl.owner
+    acl_el = SubElement(root, "AccessControlList")
+    for grant in acl.grants:
+        grant_el = SubElement(acl_el, "Grant")
+        grantee = SubElement(grant_el, "Grantee")
+        if grant.grantee == GRANTEE_ALL_USERS:
+            grantee.set("{http://www.w3.org/2001/XMLSchema-instance}type", "Group")
+            SubElement(grantee, "URI").text = "http://acs.amazonaws.com/groups/global/AllUsers"
+        elif grant.grantee == GRANTEE_AUTHENTICATED_USERS:
+            grantee.set("{http://www.w3.org/2001/XMLSchema-instance}type", "Group")
+            SubElement(grantee, "URI").text = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
+        else:
+            grantee.set("{http://www.w3.org/2001/XMLSchema-instance}type", "CanonicalUser")
+            SubElement(grantee, "ID").text = grant.grantee
+            SubElement(grantee, "DisplayName").text = grant.grantee
+        SubElement(grant_el, "Permission").text = grant.permission
+    return _xml_response(root)
+
+
+def _object_attributes_handler(bucket_name: str, object_key: str) -> Response:
+    if request.method != "GET":
+        return _method_not_allowed(["GET"])
+    principal, error = _object_principal("read", bucket_name, object_key)
+    if error:
+        return error
+    storage = _storage()
+    try:
+        path = storage.get_object_path(bucket_name, object_key)
+        file_stat = path.stat()
+        metadata = storage.get_object_metadata(bucket_name, object_key)
+    except (StorageError, FileNotFoundError):
+        return _error_response("NoSuchKey", "Object not found", 404)
+
+    requested = request.headers.get("x-amz-object-attributes", "")
+    attrs = {a.strip() for a in requested.split(",") if a.strip()}
+
+    root = Element("GetObjectAttributesResponse")
+    if "ETag" in attrs:
+        etag = metadata.get("__etag__") or storage._compute_etag(path)
+        SubElement(root, "ETag").text = etag
+    if "StorageClass" in attrs:
+        SubElement(root, "StorageClass").text = "STANDARD"
+    if "ObjectSize" in attrs:
+        SubElement(root, "ObjectSize").text = str(file_stat.st_size)
+    if "Checksum" in attrs:
+        SubElement(root, "Checksum")
+    if "ObjectParts" in attrs:
+        SubElement(root, "ObjectParts")
+
+    response = _xml_response(root)
+    response.headers["Last-Modified"] = http_date(file_stat.st_mtime)
+    return response
+
+
 def _bucket_list_versions_handler(bucket_name: str) -> Response:
     """Handle ListObjectVersions (GET /<bucket>?versions)."""
     if request.method != "GET":
@@ -2360,6 +2481,10 @@ def _post_object(bucket_name: str) -> Response:
     if success_action_redirect:
         allowed_hosts = current_app.config.get("ALLOWED_REDIRECT_HOSTS", [])
         if not allowed_hosts:
+            current_app.logger.warning(
+                "ALLOWED_REDIRECT_HOSTS not configured, falling back to request Host header. "
+                "Set ALLOWED_REDIRECT_HOSTS for production deployments."
+            )
             allowed_hosts = [request.host]
         parsed = urlparse(success_action_redirect)
         if parsed.scheme not in ("http", "https"):
@@ -2546,54 +2671,43 @@ def bucket_handler(bucket_name: str) -> Response:
     else:
         effective_start = marker
     
-    fetch_keys = max_keys * 10 if delimiter else max_keys
     try:
-        list_result = storage.list_objects(
-            bucket_name,
-            max_keys=fetch_keys,
-            continuation_token=effective_start or None,
-            prefix=prefix or None,
-        )
-        objects = list_result.objects
+        if delimiter:
+            shallow_result = storage.list_objects_shallow(
+                bucket_name,
+                prefix=prefix,
+                delimiter=delimiter,
+                max_keys=max_keys,
+                continuation_token=effective_start or None,
+            )
+            objects = shallow_result.objects
+            common_prefixes = shallow_result.common_prefixes
+            is_truncated = shallow_result.is_truncated
+
+            next_marker = shallow_result.next_continuation_token or ""
+            next_continuation_token = ""
+            if is_truncated and next_marker and list_type == "2":
+                next_continuation_token = base64.urlsafe_b64encode(next_marker.encode()).decode("utf-8")
+        else:
+            list_result = storage.list_objects(
+                bucket_name,
+                max_keys=max_keys,
+                continuation_token=effective_start or None,
+                prefix=prefix or None,
+            )
+            objects = list_result.objects
+            common_prefixes = []
+            is_truncated = list_result.is_truncated
+
+            next_marker = ""
+            next_continuation_token = ""
+            if is_truncated:
+                if objects:
+                    next_marker = objects[-1].key
+                if list_type == "2" and next_marker:
+                    next_continuation_token = base64.urlsafe_b64encode(next_marker.encode()).decode("utf-8")
     except StorageError as exc:
         return _error_response("NoSuchBucket", str(exc), 404)
-    
-    common_prefixes: list[str] = []
-    filtered_objects: list = []
-    if delimiter:
-        seen_prefixes: set[str] = set()
-        for obj in objects:
-            key_after_prefix = obj.key[len(prefix):] if prefix else obj.key
-            if delimiter in key_after_prefix:
-                common_prefix = prefix + key_after_prefix.split(delimiter)[0] + delimiter
-                if common_prefix not in seen_prefixes:
-                    seen_prefixes.add(common_prefix)
-                    common_prefixes.append(common_prefix)
-            else:
-                filtered_objects.append(obj)
-        objects = filtered_objects
-        common_prefixes = sorted(common_prefixes)
-    
-    total_items = len(objects) + len(common_prefixes)
-    is_truncated = total_items > max_keys or list_result.is_truncated
-    
-    if len(objects) >= max_keys:
-        objects = objects[:max_keys]
-        common_prefixes = []
-    else:
-        remaining = max_keys - len(objects)
-        common_prefixes = common_prefixes[:remaining]
-    
-    next_marker = ""
-    next_continuation_token = ""
-    if is_truncated:
-        if objects:
-            next_marker = objects[-1].key
-        elif common_prefixes:
-            next_marker = common_prefixes[-1].rstrip(delimiter) if delimiter else common_prefixes[-1]
-        
-        if list_type == "2" and next_marker:
-            next_continuation_token = base64.urlsafe_b64encode(next_marker.encode()).decode("utf-8")
 
     if list_type == "2":
         root = Element("ListBucketResult")
@@ -2669,6 +2783,12 @@ def object_handler(bucket_name: str, object_key: str):
     if "legal-hold" in request.args:
         return _object_legal_hold_handler(bucket_name, object_key)
 
+    if "acl" in request.args:
+        return _object_acl_handler(bucket_name, object_key)
+
+    if "attributes" in request.args:
+        return _object_attributes_handler(bucket_name, object_key)
+
     if request.method == "POST":
         if "uploads" in request.args:
             return _initiate_multipart_upload(bucket_name, object_key)
@@ -2816,7 +2936,7 @@ def object_handler(bucket_name: str, object_key: str):
                             f.seek(start_pos)
                             remaining = length_to_read
                             while remaining > 0:
-                                chunk_size = min(65536, remaining)
+                                chunk_size = min(262144, remaining)
                                 chunk = f.read(chunk_size)
                                 if not chunk:
                                     break
@@ -2993,6 +3113,32 @@ def _bucket_policy_handler(bucket_name: str) -> Response:
     return Response(status=204)
 
 
+def _bucket_policy_status_handler(bucket_name: str) -> Response:
+    if request.method != "GET":
+        return _method_not_allowed(["GET"])
+    principal, error = _require_principal()
+    if error:
+        return error
+    try:
+        _authorize_action(principal, bucket_name, "policy")
+    except IamError as exc:
+        return _error_response("AccessDenied", str(exc), 403)
+    storage = _storage()
+    if not storage.bucket_exists(bucket_name):
+        return _error_response("NoSuchBucket", "Bucket does not exist", 404)
+    store = _bucket_policies()
+    policy = store.get_policy(bucket_name)
+    is_public = False
+    if policy:
+        for statement in policy.get("Statement", []):
+            if statement.get("Effect") == "Allow" and statement.get("Principal") == "*":
+                is_public = True
+                break
+    root = Element("PolicyStatus")
+    SubElement(root, "IsPublic").text = "TRUE" if is_public else "FALSE"
+    return _xml_response(root)
+
+
 def _bucket_replication_handler(bucket_name: str) -> Response:
     if request.method not in {"GET", "PUT", "DELETE"}:
         return _method_not_allowed(["GET", "PUT", "DELETE"])
@@ -3206,7 +3352,7 @@ def head_object(bucket_name: str, object_key: str) -> Response:
         path = _storage().get_object_path(bucket_name, object_key)
         metadata = _storage().get_object_metadata(bucket_name, object_key)
         stat = path.stat()
-        etag = _storage()._compute_etag(path)
+        etag = metadata.get("__etag__") or _storage()._compute_etag(path)
         
         response = Response(status=200)
         _apply_object_headers(response, file_stat=stat, metadata=metadata, etag=etag)

app/s3_client.py

@@ -245,6 +245,7 @@ def stream_objects_ndjson(
     url_templates: dict[str, str],
     display_tz: str = "UTC",
     versioning_enabled: bool = False,
+    delimiter: Optional[str] = None,
 ) -> Generator[str, None, None]:
     meta_line = json.dumps({
         "type": "meta",
@@ -258,11 +259,20 @@ def stream_objects_ndjson(
     kwargs: dict[str, Any] = {"Bucket": bucket_name, "MaxKeys": 1000}
     if prefix:
         kwargs["Prefix"] = prefix
+    if delimiter:
+        kwargs["Delimiter"] = delimiter
 
+    running_count = 0
     try:
         paginator = client.get_paginator("list_objects_v2")
         for page in paginator.paginate(**kwargs):
-            for obj in page.get("Contents", []):
+            for cp in page.get("CommonPrefixes", []):
+                yield json.dumps({
+                    "type": "folder",
+                    "prefix": cp["Prefix"],
+                }) + "\n"
+            page_contents = page.get("Contents", [])
+            for obj in page_contents:
                 last_mod = obj["LastModified"]
                 yield json.dumps({
                     "type": "object",
@@ -273,6 +283,8 @@ def stream_objects_ndjson(
                     "last_modified_iso": format_datetime_iso(last_mod, display_tz),
                     "etag": obj.get("ETag", "").strip('"'),
                 }) + "\n"
+            running_count += len(page_contents)
+            yield json.dumps({"type": "count", "total_count": running_count}) + "\n"
     except ClientError as exc:
         error_msg = exc.response.get("Error", {}).get("Message", "S3 operation failed")
         yield json.dumps({"type": "error", "error": error_msg}) + "\n"

app/storage.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import copy
 import hashlib
 import json
 import os
@@ -153,6 +154,15 @@ class ListObjectsResult:
     total_count: Optional[int] = None
 
 
+@dataclass
+class ShallowListResult:
+    """Result for delimiter-aware directory-level listing."""
+    objects: List[ObjectMeta]
+    common_prefixes: List[str]
+    is_truncated: bool
+    next_continuation_token: Optional[str]
+
+
 def _utcnow() -> datetime:
     return datetime.now(timezone.utc)
 
@@ -196,6 +206,8 @@ class ObjectStorage:
         self._object_key_max_length_bytes = object_key_max_length_bytes
         self._sorted_key_cache: Dict[str, tuple[list[str], int]] = {}
         self._meta_index_locks: Dict[str, threading.Lock] = {}
+        self._meta_read_cache: OrderedDict[tuple, Optional[Dict[str, Any]]] = OrderedDict()
+        self._meta_read_cache_max = 2048
         self._cleanup_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="ParentCleanup")
 
     def _get_bucket_lock(self, bucket_id: str) -> threading.Lock:
@@ -276,25 +288,41 @@ class ObjectStorage:
         version_count = 0
         version_bytes = 0
 
+        internal = self.INTERNAL_FOLDERS
+        bucket_str = str(bucket_path)
+
         try:
-            for path in bucket_path.rglob("*"):
-                if path.is_file():
-                    rel = path.relative_to(bucket_path)
-                    if not rel.parts:
-                        continue
-                    top_folder = rel.parts[0]
-                    if top_folder not in self.INTERNAL_FOLDERS:
-                        stat = path.stat()
-                        object_count += 1
-                        total_bytes += stat.st_size
+            stack = [bucket_str]
+            while stack:
+                current = stack.pop()
+                try:
+                    with os.scandir(current) as it:
+                        for entry in it:
+                            if current == bucket_str and entry.name in internal:
+                                continue
+                            if entry.is_dir(follow_symlinks=False):
+                                stack.append(entry.path)
+                            elif entry.is_file(follow_symlinks=False):
+                                object_count += 1
+                                total_bytes += entry.stat(follow_symlinks=False).st_size
+                except PermissionError:
+                    continue
 
             versions_root = self._bucket_versions_root(bucket_name)
             if versions_root.exists():
-                for path in versions_root.rglob("*.bin"):
-                    if path.is_file():
-                        stat = path.stat()
-                        version_count += 1
-                        version_bytes += stat.st_size
+                v_stack = [str(versions_root)]
+                while v_stack:
+                    v_current = v_stack.pop()
+                    try:
+                        with os.scandir(v_current) as it:
+                            for entry in it:
+                                if entry.is_dir(follow_symlinks=False):
+                                    v_stack.append(entry.path)
+                                elif entry.is_file(follow_symlinks=False) and entry.name.endswith(".bin"):
+                                    version_count += 1
+                                    version_bytes += entry.stat(follow_symlinks=False).st_size
+                    except PermissionError:
+                        continue
         except OSError:
             if cached_stats is not None:
                 return cached_stats
@@ -374,9 +402,18 @@ class ObjectStorage:
             raise StorageError("Bucket contains archived object versions")
         if has_multipart:
             raise StorageError("Bucket has active multipart uploads")
+        bucket_id = bucket_path.name
         self._remove_tree(bucket_path)
-        self._remove_tree(self._system_bucket_root(bucket_path.name))
-        self._remove_tree(self._multipart_bucket_root(bucket_path.name))
+        self._remove_tree(self._system_bucket_root(bucket_id))
+        self._remove_tree(self._multipart_bucket_root(bucket_id))
+        self._bucket_config_cache.pop(bucket_id, None)
+        with self._cache_lock:
+            self._object_cache.pop(bucket_id, None)
+            self._cache_version.pop(bucket_id, None)
+            self._sorted_key_cache.pop(bucket_id, None)
+            stale = [k for k in self._meta_read_cache if k[0] == bucket_id]
+            for k in stale:
+                del self._meta_read_cache[k]
 
     def list_objects(
         self,
@@ -459,6 +496,279 @@ class ObjectStorage:
         result = self.list_objects(bucket_name, max_keys=100000)
         return result.objects
 
+    def list_objects_shallow(
+        self,
+        bucket_name: str,
+        *,
+        prefix: str = "",
+        delimiter: str = "/",
+        max_keys: int = 1000,
+        continuation_token: Optional[str] = None,
+    ) -> ShallowListResult:
+        import bisect
+
+        bucket_path = self._bucket_path(bucket_name)
+        if not bucket_path.exists():
+            raise BucketNotFoundError("Bucket does not exist")
+        bucket_id = bucket_path.name
+
+        if delimiter != "/" or (prefix and not prefix.endswith(delimiter)):
+            return self._shallow_via_full_scan(
+                bucket_name, prefix=prefix, delimiter=delimiter,
+                max_keys=max_keys, continuation_token=continuation_token,
+            )
+
+        target_dir = bucket_path
+        if prefix:
+            safe_prefix_path = Path(prefix.rstrip("/"))
+            if ".." in safe_prefix_path.parts:
+                return ShallowListResult(
+                    objects=[], common_prefixes=[],
+                    is_truncated=False, next_continuation_token=None,
+                )
+            target_dir = bucket_path / safe_prefix_path
+            try:
+                resolved = target_dir.resolve()
+                bucket_resolved = bucket_path.resolve()
+                if not str(resolved).startswith(str(bucket_resolved) + os.sep) and resolved != bucket_resolved:
+                    return ShallowListResult(
+                        objects=[], common_prefixes=[],
+                        is_truncated=False, next_continuation_token=None,
+                    )
+            except (OSError, ValueError):
+                return ShallowListResult(
+                    objects=[], common_prefixes=[],
+                    is_truncated=False, next_continuation_token=None,
+                )
+
+        if not target_dir.exists() or not target_dir.is_dir():
+            return ShallowListResult(
+                objects=[], common_prefixes=[],
+                is_truncated=False, next_continuation_token=None,
+            )
+
+        etag_index_path = self._system_bucket_root(bucket_id) / "etag_index.json"
+        meta_cache: Dict[str, str] = {}
+        if etag_index_path.exists():
+            try:
+                with open(etag_index_path, 'r', encoding='utf-8') as f:
+                    meta_cache = json.load(f)
+            except (OSError, json.JSONDecodeError):
+                pass
+
+        entries_files: list[tuple[str, int, float, Optional[str]]] = []
+        entries_dirs: list[str] = []
+
+        try:
+            with os.scandir(str(target_dir)) as it:
+                for entry in it:
+                    name = entry.name
+                    if name in self.INTERNAL_FOLDERS:
+                        continue
+                    if entry.is_dir(follow_symlinks=False):
+                        cp = prefix + name + delimiter
+                        entries_dirs.append(cp)
+                    elif entry.is_file(follow_symlinks=False):
+                        key = prefix + name
+                        try:
+                            st = entry.stat()
+                            etag = meta_cache.get(key)
+                            entries_files.append((key, st.st_size, st.st_mtime, etag))
+                        except OSError:
+                            pass
+        except OSError:
+            return ShallowListResult(
+                objects=[], common_prefixes=[],
+                is_truncated=False, next_continuation_token=None,
+            )
+
+        entries_dirs.sort()
+        entries_files.sort(key=lambda x: x[0])
+
+        all_items: list[tuple[str, bool]] = []
+        fi, di = 0, 0
+        while fi < len(entries_files) and di < len(entries_dirs):
+            if entries_files[fi][0] <= entries_dirs[di]:
+                all_items.append((entries_files[fi][0], False))
+                fi += 1
+            else:
+                all_items.append((entries_dirs[di], True))
+                di += 1
+        while fi < len(entries_files):
+            all_items.append((entries_files[fi][0], False))
+            fi += 1
+        while di < len(entries_dirs):
+            all_items.append((entries_dirs[di], True))
+            di += 1
+
+        files_map = {e[0]: e for e in entries_files}
+
+        start_index = 0
+        if continuation_token:
+            all_keys = [item[0] for item in all_items]
+            start_index = bisect.bisect_right(all_keys, continuation_token)
+
+        selected = all_items[start_index:start_index + max_keys]
+        is_truncated = (start_index + max_keys) < len(all_items)
+
+        result_objects: list[ObjectMeta] = []
+        result_prefixes: list[str] = []
+        for item_key, is_dir in selected:
+            if is_dir:
+                result_prefixes.append(item_key)
+            else:
+                fdata = files_map[item_key]
+                result_objects.append(ObjectMeta(
+                    key=fdata[0],
+                    size=fdata[1],
+                    last_modified=datetime.fromtimestamp(fdata[2], timezone.utc),
+                    etag=fdata[3],
+                    metadata=None,
+                ))
+
+        next_token = None
+        if is_truncated and selected:
+            next_token = selected[-1][0]
+
+        return ShallowListResult(
+            objects=result_objects,
+            common_prefixes=result_prefixes,
+            is_truncated=is_truncated,
+            next_continuation_token=next_token,
+        )
+
+    def _shallow_via_full_scan(
+        self,
+        bucket_name: str,
+        *,
+        prefix: str = "",
+        delimiter: str = "/",
+        max_keys: int = 1000,
+        continuation_token: Optional[str] = None,
+    ) -> ShallowListResult:
+        list_result = self.list_objects(
+            bucket_name,
+            max_keys=max_keys * 10,
+            continuation_token=continuation_token,
+            prefix=prefix or None,
+        )
+
+        common_prefixes: list[str] = []
+        filtered_objects: list[ObjectMeta] = []
+        seen_prefixes: set[str] = set()
+
+        for obj in list_result.objects:
+            key_after_prefix = obj.key[len(prefix):] if prefix else obj.key
+            if delimiter in key_after_prefix:
+                cp = prefix + key_after_prefix.split(delimiter)[0] + delimiter
+                if cp not in seen_prefixes:
+                    seen_prefixes.add(cp)
+                    common_prefixes.append(cp)
+            else:
+                filtered_objects.append(obj)
+
+        common_prefixes.sort()
+        total_items = len(filtered_objects) + len(common_prefixes)
+        is_truncated = total_items > max_keys or list_result.is_truncated
+
+        if len(filtered_objects) >= max_keys:
+            filtered_objects = filtered_objects[:max_keys]
+            common_prefixes = []
+        else:
+            remaining = max_keys - len(filtered_objects)
+            common_prefixes = common_prefixes[:remaining]
+
+        next_token = None
+        if is_truncated:
+            if filtered_objects:
+                next_token = filtered_objects[-1].key
+            elif common_prefixes:
+                next_token = common_prefixes[-1].rstrip(delimiter) if delimiter else common_prefixes[-1]
+
+        return ShallowListResult(
+            objects=filtered_objects,
+            common_prefixes=common_prefixes,
+            is_truncated=is_truncated,
+            next_continuation_token=next_token,
+        )
+
+    def search_objects(
+        self,
+        bucket_name: str,
+        query: str,
+        *,
+        prefix: str = "",
+        limit: int = 500,
+    ) -> Dict[str, Any]:
+        bucket_path = self._bucket_path(bucket_name)
+        if not bucket_path.is_dir():
+            raise BucketNotFoundError("Bucket does not exist")
+
+        if prefix:
+            search_root = bucket_path / prefix.replace("/", os.sep)
+            if not search_root.is_dir():
+                return {"results": [], "truncated": False}
+            resolved = search_root.resolve()
+            if not str(resolved).startswith(str(bucket_path.resolve())):
+                return {"results": [], "truncated": False}
+        else:
+            search_root = bucket_path
+
+        query_lower = query.lower()
+        results: list[Dict[str, Any]] = []
+        internal = self.INTERNAL_FOLDERS
+        bucket_str = str(bucket_path)
+        bucket_len = len(bucket_str) + 1
+        meta_root = self._bucket_meta_root(bucket_name)
+        scan_limit = limit * 4
+
+        matched = 0
+        scanned = 0
+        search_str = str(search_root)
+        stack = [search_str]
+        while stack:
+            current = stack.pop()
+            try:
+                with os.scandir(current) as it:
+                    for entry in it:
+                        if current == bucket_str and entry.name in internal:
+                            continue
+                        if entry.is_dir(follow_symlinks=False):
+                            stack.append(entry.path)
+                        elif entry.is_file(follow_symlinks=False):
+                            scanned += 1
+                            key = entry.path[bucket_len:].replace(os.sep, "/")
+                            if query_lower in key.lower():
+                                st = entry.stat(follow_symlinks=False)
+                                meta_path = meta_root / (key + ".meta.json")
+                                last_modified = ""
+                                try:
+                                    if meta_path.exists():
+                                        md = json.loads(meta_path.read_text(encoding="utf-8"))
+                                        last_modified = md.get("last_modified", "")
+                                except (OSError, json.JSONDecodeError):
+                                    pass
+                                if not last_modified:
+                                    last_modified = datetime.fromtimestamp(
+                                        st.st_mtime, tz=timezone.utc
+                                    ).strftime("%Y-%m-%dT%H:%M:%S.000Z")
+                                results.append({
+                                    "key": key,
+                                    "size": st.st_size,
+                                    "last_modified": last_modified,
+                                })
+                                matched += 1
+                            if matched >= scan_limit:
+                                break
+            except PermissionError:
+                continue
+            if matched >= scan_limit:
+                break
+
+        results.sort(key=lambda r: r["key"])
+        truncated = len(results) > limit
+        return {"results": results[:limit], "truncated": truncated}
+
     def put_object(
         self,
         bucket_name: str,
@@ -959,13 +1269,19 @@ class ObjectStorage:
             version_bytes_delta=archived_version_size,
             version_count_delta=1 if archived_version_size > 0 else 0,
         )
-        return ObjectMeta(
+        etag = self._compute_etag(destination)
+        internal_meta = {"__etag__": etag, "__size__": str(stat.st_size)}
+        combined_meta = {**internal_meta, **(metadata or {})}
+        self._write_metadata(bucket_id, safe_key, combined_meta)
+        obj_meta = ObjectMeta(
             key=safe_key.as_posix(),
             size=stat.st_size,
             last_modified=datetime.fromtimestamp(stat.st_mtime, timezone.utc),
-            etag=self._compute_etag(destination),
+            etag=etag,
             metadata=metadata or None,
         )
+        self._update_object_cache_entry(bucket_id, safe_key.as_posix(), obj_meta)
+        return obj_meta
 
     def delete_object_version(self, bucket_name: str, object_key: str, version_id: str) -> None:
         bucket_path = self._bucket_path(bucket_name)
@@ -1763,11 +2079,6 @@ class ObjectStorage:
             return 0
 
     def _update_object_cache_entry(self, bucket_id: str, key: str, meta: Optional[ObjectMeta]) -> None:
-        """Update a single entry in the object cache instead of invalidating the whole cache.
-
-        This is a performance optimization - lazy update instead of full invalidation.
-        Cross-process invalidation is handled by checking stats.json mtime.
-        """
         with self._cache_lock:
             cached = self._object_cache.get(bucket_id)
             if cached:
@@ -1779,6 +2090,25 @@ class ObjectStorage:
             self._cache_version[bucket_id] = self._cache_version.get(bucket_id, 0) + 1
             self._sorted_key_cache.pop(bucket_id, None)
 
+        self._update_etag_index(bucket_id, key, meta.etag if meta else None)
+
+    def _update_etag_index(self, bucket_id: str, key: str, etag: Optional[str]) -> None:
+        etag_index_path = self._system_bucket_root(bucket_id) / "etag_index.json"
+        try:
+            index: Dict[str, str] = {}
+            if etag_index_path.exists():
+                with open(etag_index_path, 'r', encoding='utf-8') as f:
+                    index = json.load(f)
+            if etag is None:
+                index.pop(key, None)
+            else:
+                index[key] = etag
+            etag_index_path.parent.mkdir(parents=True, exist_ok=True)
+            with open(etag_index_path, 'w', encoding='utf-8') as f:
+                json.dump(index, f)
+        except (OSError, json.JSONDecodeError):
+            pass
+
     def warm_cache(self, bucket_names: Optional[List[str]] = None) -> None:
         """Pre-warm the object cache for specified buckets or all buckets.
 
@@ -1829,30 +2159,40 @@ class ObjectStorage:
 
     def _read_bucket_config(self, bucket_name: str) -> dict[str, Any]:
         now = time.time()
+        config_path = self._bucket_config_path(bucket_name)
         cached = self._bucket_config_cache.get(bucket_name)
         if cached:
-            config, cached_time = cached
+            config, cached_time, cached_mtime = cached
             if now - cached_time < self._bucket_config_cache_ttl:
-                return config.copy()
+                try:
+                    current_mtime = config_path.stat().st_mtime if config_path.exists() else 0.0
+                except OSError:
+                    current_mtime = 0.0
+                if current_mtime == cached_mtime:
+                    return config.copy()
 
-        config_path = self._bucket_config_path(bucket_name)
         if not config_path.exists():
-            self._bucket_config_cache[bucket_name] = ({}, now)
+            self._bucket_config_cache[bucket_name] = ({}, now, 0.0)
             return {}
         try:
             data = json.loads(config_path.read_text(encoding="utf-8"))
             config = data if isinstance(data, dict) else {}
-            self._bucket_config_cache[bucket_name] = (config, now)
+            mtime = config_path.stat().st_mtime
+            self._bucket_config_cache[bucket_name] = (config, now, mtime)
             return config.copy()
         except (OSError, json.JSONDecodeError):
-            self._bucket_config_cache[bucket_name] = ({}, now)
+            self._bucket_config_cache[bucket_name] = ({}, now, 0.0)
             return {}
 
     def _write_bucket_config(self, bucket_name: str, payload: dict[str, Any]) -> None:
         config_path = self._bucket_config_path(bucket_name)
         config_path.parent.mkdir(parents=True, exist_ok=True)
         config_path.write_text(json.dumps(payload), encoding="utf-8")
-        self._bucket_config_cache[bucket_name] = (payload.copy(), time.time())
+        try:
+            mtime = config_path.stat().st_mtime
+        except OSError:
+            mtime = 0.0
+        self._bucket_config_cache[bucket_name] = (payload.copy(), time.time(), mtime)
 
     def _set_bucket_config_entry(self, bucket_name: str, key: str, value: Any | None) -> None:
         config = self._read_bucket_config(bucket_name)
@@ -1904,14 +2244,38 @@ class ObjectStorage:
             return self._meta_index_locks[index_path]
 
     def _read_index_entry(self, bucket_name: str, key: Path) -> Optional[Dict[str, Any]]:
+        cache_key = (bucket_name, str(key))
+        with self._cache_lock:
+            hit = self._meta_read_cache.get(cache_key)
+            if hit is not None:
+                self._meta_read_cache.move_to_end(cache_key)
+                cached = hit[0]
+                return copy.deepcopy(cached) if cached is not None else None
+
         index_path, entry_name = self._index_file_for_key(bucket_name, key)
-        if not index_path.exists():
-            return None
-        try:
-            index_data = json.loads(index_path.read_text(encoding="utf-8"))
-            return index_data.get(entry_name)
-        except (OSError, json.JSONDecodeError):
-            return None
+        if _HAS_RUST:
+            result = _rc.read_index_entry(str(index_path), entry_name)
+        else:
+            if not index_path.exists():
+                result = None
+            else:
+                try:
+                    index_data = json.loads(index_path.read_text(encoding="utf-8"))
+                    result = index_data.get(entry_name)
+                except (OSError, json.JSONDecodeError):
+                    result = None
+
+        with self._cache_lock:
+            while len(self._meta_read_cache) >= self._meta_read_cache_max:
+                self._meta_read_cache.popitem(last=False)
+            self._meta_read_cache[cache_key] = (copy.deepcopy(result) if result is not None else None,)
+
+        return result
+
+    def _invalidate_meta_read_cache(self, bucket_name: str, key: Path) -> None:
+        cache_key = (bucket_name, str(key))
+        with self._cache_lock:
+            self._meta_read_cache.pop(cache_key, None)
 
     def _write_index_entry(self, bucket_name: str, key: Path, entry: Dict[str, Any]) -> None:
         index_path, entry_name = self._index_file_for_key(bucket_name, key)
@@ -1926,16 +2290,19 @@ class ObjectStorage:
                     pass
             index_data[entry_name] = entry
             index_path.write_text(json.dumps(index_data), encoding="utf-8")
+        self._invalidate_meta_read_cache(bucket_name, key)
 
     def _delete_index_entry(self, bucket_name: str, key: Path) -> None:
         index_path, entry_name = self._index_file_for_key(bucket_name, key)
         if not index_path.exists():
+            self._invalidate_meta_read_cache(bucket_name, key)
             return
         lock = self._get_meta_index_lock(str(index_path))
         with lock:
             try:
                 index_data = json.loads(index_path.read_text(encoding="utf-8"))
             except (OSError, json.JSONDecodeError):
+                self._invalidate_meta_read_cache(bucket_name, key)
                 return
             if entry_name in index_data:
                 del index_data[entry_name]
@@ -1946,6 +2313,7 @@ class ObjectStorage:
                         index_path.unlink()
                     except OSError:
                         pass
+        self._invalidate_meta_read_cache(bucket_name, key)
 
     def _normalize_metadata(self, metadata: Optional[Dict[str, str]]) -> Optional[Dict[str, str]]:
         if not metadata:

app/ui.py

@@ -508,11 +508,15 @@ def bucket_detail(bucket_name: str):
     can_manage_quota = is_replication_admin
 
     website_config = None
+    website_domains = []
     if website_hosting_enabled:
         try:
             website_config = storage.get_bucket_website(bucket_name)
         except StorageError:
             website_config = None
+        domain_store = current_app.extensions.get("website_domains")
+        if domain_store:
+            website_domains = domain_store.get_domains_for_bucket(bucket_name)
 
     objects_api_url = url_for("ui.list_bucket_objects", bucket_name=bucket_name)
     objects_stream_url = url_for("ui.stream_bucket_objects", bucket_name=bucket_name)
@@ -558,6 +562,7 @@ def bucket_detail(bucket_name: str):
         site_sync_enabled=site_sync_enabled,
         website_hosting_enabled=website_hosting_enabled,
         website_config=website_config,
+        website_domains=website_domains,
         can_manage_website=can_edit_policy,
     )
 
@@ -611,6 +616,7 @@ def stream_bucket_objects(bucket_name: str):
         return jsonify({"error": str(exc)}), 403
 
     prefix = request.args.get("prefix") or None
+    delimiter = request.args.get("delimiter") or None
 
     try:
         client = get_session_s3_client()
@@ -624,6 +630,7 @@ def stream_bucket_objects(bucket_name: str):
     return Response(
         stream_objects_ndjson(
             client, bucket_name, prefix, url_templates, display_tz, versioning_enabled,
+            delimiter=delimiter,
         ),
         mimetype='application/x-ndjson',
         headers={
@@ -634,6 +641,33 @@ def stream_bucket_objects(bucket_name: str):
     )
 
 
+@ui_bp.get("/buckets/<bucket_name>/objects/search")
+@limiter.limit("30 per minute")
+def search_bucket_objects(bucket_name: str):
+    principal = _current_principal()
+    try:
+        _authorize_ui(principal, bucket_name, "list")
+    except IamError as exc:
+        return jsonify({"error": str(exc)}), 403
+
+    query = request.args.get("q", "").strip()
+    if not query:
+        return jsonify({"results": [], "truncated": False})
+
+    try:
+        limit = max(1, min(int(request.args.get("limit", 500)), 1000))
+    except (ValueError, TypeError):
+        limit = 500
+
+    prefix = request.args.get("prefix", "").strip()
+
+    storage = _storage()
+    try:
+        return jsonify(storage.search_objects(bucket_name, query, prefix=prefix, limit=limit))
+    except StorageError as exc:
+        return jsonify({"error": str(exc)}), 404
+
+
 @ui_bp.post("/buckets/<bucket_name>/upload")
 @limiter.limit("30 per minute")
 def upload_object(bucket_name: str):
@@ -738,7 +772,6 @@ def initiate_multipart_upload(bucket_name: str):
 
 
 @ui_bp.put("/buckets/<bucket_name>/multipart/<upload_id>/parts")
-@limiter.exempt
 @csrf.exempt
 def upload_multipart_part(bucket_name: str, upload_id: str):
     principal = _current_principal()
@@ -1297,12 +1330,14 @@ def object_versions(bucket_name: str, object_key: str):
         for v in resp.get("Versions", []):
             if v.get("Key") != object_key:
                 continue
+            if v.get("IsLatest", False):
+                continue
             versions.append({
                 "version_id": v.get("VersionId", ""),
                 "last_modified": v["LastModified"].isoformat() if v.get("LastModified") else None,
                 "size": v.get("Size", 0),
                 "etag": v.get("ETag", "").strip('"'),
-                "is_latest": v.get("IsLatest", False),
+                "is_latest": False,
             })
         return jsonify({"versions": versions})
     except (ClientError, EndpointConnectionError, ConnectionClosedError) as exc:
@@ -2374,7 +2409,10 @@ def website_domains_dashboard():
     store = current_app.extensions.get("website_domains")
     mappings = store.list_all() if store else []
     storage = _storage()
-    buckets = [b.name for b in storage.list_buckets()]
+    buckets = [
+        b.name for b in storage.list_buckets()
+        if storage.get_bucket_website(b.name)
+    ]
     return render_template(
         "website_domains.html",
         mappings=mappings,
@@ -3293,9 +3331,12 @@ def sites_dashboard():
 @ui_bp.post("/sites/local")
 def update_local_site():
     principal = _current_principal()
+    wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest"
     try:
         _iam().authorize(principal, None, "iam:*")
     except IamError:
+        if wants_json:
+            return jsonify({"error": "Access denied"}), 403
         flash("Access denied", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3306,6 +3347,8 @@ def update_local_site():
     display_name = request.form.get("display_name", "").strip()
 
     if not site_id:
+        if wants_json:
+            return jsonify({"error": "Site ID is required"}), 400
         flash("Site ID is required", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3327,6 +3370,8 @@ def update_local_site():
     )
     registry.set_local_site(site)
 
+    if wants_json:
+        return jsonify({"message": "Local site configuration updated"})
     flash("Local site configuration updated", "success")
     return redirect(url_for("ui.sites_dashboard"))
 
@@ -3334,9 +3379,12 @@ def update_local_site():
 @ui_bp.post("/sites/peers")
 def add_peer_site():
     principal = _current_principal()
+    wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest"
     try:
         _iam().authorize(principal, None, "iam:*")
     except IamError:
+        if wants_json:
+            return jsonify({"error": "Access denied"}), 403
         flash("Access denied", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3348,9 +3396,13 @@ def add_peer_site():
     connection_id = request.form.get("connection_id", "").strip() or None
 
     if not site_id:
+        if wants_json:
+            return jsonify({"error": "Site ID is required"}), 400
         flash("Site ID is required", "danger")
         return redirect(url_for("ui.sites_dashboard"))
     if not endpoint:
+        if wants_json:
+            return jsonify({"error": "Endpoint is required"}), 400
         flash("Endpoint is required", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3362,10 +3414,14 @@ def add_peer_site():
     registry = _site_registry()
 
     if registry.get_peer(site_id):
+        if wants_json:
+            return jsonify({"error": f"Peer site '{site_id}' already exists"}), 409
         flash(f"Peer site '{site_id}' already exists", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
     if connection_id and not _connections().get(connection_id):
+        if wants_json:
+            return jsonify({"error": f"Connection '{connection_id}' not found"}), 404
         flash(f"Connection '{connection_id}' not found", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3379,6 +3435,11 @@ def add_peer_site():
     )
     registry.add_peer(peer)
 
+    if wants_json:
+        redirect_url = None
+        if connection_id:
+            redirect_url = url_for("ui.replication_wizard", site_id=site_id)
+        return jsonify({"message": f"Peer site '{site_id}' added", "redirect": redirect_url})
     flash(f"Peer site '{site_id}' added", "success")
 
     if connection_id:
@@ -3389,9 +3450,12 @@ def add_peer_site():
 @ui_bp.post("/sites/peers/<site_id>/update")
 def update_peer_site(site_id: str):
     principal = _current_principal()
+    wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest"
     try:
         _iam().authorize(principal, None, "iam:*")
     except IamError:
+        if wants_json:
+            return jsonify({"error": "Access denied"}), 403
         flash("Access denied", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3399,6 +3463,8 @@ def update_peer_site(site_id: str):
     existing = registry.get_peer(site_id)
 
     if not existing:
+        if wants_json:
+            return jsonify({"error": f"Peer site '{site_id}' not found"}), 404
         flash(f"Peer site '{site_id}' not found", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3406,7 +3472,10 @@ def update_peer_site(site_id: str):
     region = request.form.get("region", existing.region).strip()
     priority = request.form.get("priority", str(existing.priority))
     display_name = request.form.get("display_name", existing.display_name).strip()
-    connection_id = request.form.get("connection_id", "").strip() or existing.connection_id
+    if "connection_id" in request.form:
+        connection_id = request.form["connection_id"].strip() or None
+    else:
+        connection_id = existing.connection_id
 
     try:
         priority_int = int(priority)
@@ -3414,6 +3483,8 @@ def update_peer_site(site_id: str):
         priority_int = existing.priority
 
     if connection_id and not _connections().get(connection_id):
+        if wants_json:
+            return jsonify({"error": f"Connection '{connection_id}' not found"}), 404
         flash(f"Connection '{connection_id}' not found", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3430,6 +3501,8 @@ def update_peer_site(site_id: str):
     )
     registry.update_peer(peer)
 
+    if wants_json:
+        return jsonify({"message": f"Peer site '{site_id}' updated"})
     flash(f"Peer site '{site_id}' updated", "success")
     return redirect(url_for("ui.sites_dashboard"))
 
@@ -3437,16 +3510,23 @@ def update_peer_site(site_id: str):
 @ui_bp.post("/sites/peers/<site_id>/delete")
 def delete_peer_site(site_id: str):
     principal = _current_principal()
+    wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest"
     try:
         _iam().authorize(principal, None, "iam:*")
     except IamError:
+        if wants_json:
+            return jsonify({"error": "Access denied"}), 403
         flash("Access denied", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
     registry = _site_registry()
     if registry.delete_peer(site_id):
+        if wants_json:
+            return jsonify({"message": f"Peer site '{site_id}' deleted"})
         flash(f"Peer site '{site_id}' deleted", "success")
     else:
+        if wants_json:
+            return jsonify({"error": f"Peer site '{site_id}' not found"}), 404
         flash(f"Peer site '{site_id}' not found", "danger")
 
     return redirect(url_for("ui.sites_dashboard"))

app/version.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-APP_VERSION = "0.3.0"
+APP_VERSION = "0.3.2"
 
 
 def get_version() -> str:

app/website_domains.py

@@ -35,13 +35,16 @@ class WebsiteDomainStore:
         self.config_path = config_path
         self._lock = threading.Lock()
         self._domains: Dict[str, str] = {}
+        self._last_mtime: float = 0.0
         self.reload()
 
     def reload(self) -> None:
         if not self.config_path.exists():
             self._domains = {}
+            self._last_mtime = 0.0
             return
         try:
+            self._last_mtime = self.config_path.stat().st_mtime
             with open(self.config_path, "r", encoding="utf-8") as f:
                 data = json.load(f)
                 if isinstance(data, dict):
@@ -51,19 +54,45 @@ class WebsiteDomainStore:
         except (OSError, json.JSONDecodeError):
             self._domains = {}
 
+    def _maybe_reload(self) -> None:
+        try:
+            if self.config_path.exists():
+                mtime = self.config_path.stat().st_mtime
+                if mtime != self._last_mtime:
+                    self._last_mtime = mtime
+                    with open(self.config_path, "r", encoding="utf-8") as f:
+                        data = json.load(f)
+                    if isinstance(data, dict):
+                        self._domains = {k.lower(): v for k, v in data.items()}
+                    else:
+                        self._domains = {}
+            elif self._domains:
+                self._domains = {}
+                self._last_mtime = 0.0
+        except (OSError, json.JSONDecodeError):
+            pass
+
     def _save(self) -> None:
         self.config_path.parent.mkdir(parents=True, exist_ok=True)
         with open(self.config_path, "w", encoding="utf-8") as f:
             json.dump(self._domains, f, indent=2)
+        self._last_mtime = self.config_path.stat().st_mtime
 
     def list_all(self) -> List[Dict[str, str]]:
         with self._lock:
+            self._maybe_reload()
             return [{"domain": d, "bucket": b} for d, b in self._domains.items()]
 
     def get_bucket(self, domain: str) -> Optional[str]:
         with self._lock:
+            self._maybe_reload()
             return self._domains.get(domain.lower())
 
+    def get_domains_for_bucket(self, bucket: str) -> List[str]:
+        with self._lock:
+            self._maybe_reload()
+            return [d for d, b in self._domains.items() if b == bucket]
+
     def set_mapping(self, domain: str, bucket: str) -> None:
         with self._lock:
             self._domains[domain.lower()] = bucket

docs.md

@@ -139,6 +139,7 @@ All configuration is done via environment variables. The table below lists every
 | `API_BASE_URL` | `http://127.0.0.1:5000` | Internal S3 API URL used by the web UI proxy. Also used for presigned URL generation. Set to your public URL if running behind a reverse proxy. |
 | `AWS_REGION` | `us-east-1` | Region embedded in SigV4 credential scope. |
 | `AWS_SERVICE` | `s3` | Service string for SigV4. |
+| `DISPLAY_TIMEZONE` | `UTC` | Timezone for timestamps in the web UI (e.g., `US/Eastern`, `Asia/Tokyo`). |
 
 ### IAM & Security
 
@@ -170,6 +171,7 @@ All configuration is done via environment variables. The table below lists every
 | `RATE_LIMIT_BUCKET_OPS` | `120 per minute` | Rate limit for bucket operations (PUT/DELETE/GET/POST on `/<bucket>`). |
 | `RATE_LIMIT_OBJECT_OPS` | `240 per minute` | Rate limit for object operations (PUT/GET/DELETE/POST on `/<bucket>/<key>`). |
 | `RATE_LIMIT_HEAD_OPS` | `100 per minute` | Rate limit for HEAD requests (bucket and object). |
+| `RATE_LIMIT_ADMIN` | `60 per minute` | Rate limit for admin API endpoints (`/admin/*`). |
 | `RATE_LIMIT_STORAGE_URI` | `memory://` | Storage backend for rate limits. Use `redis://host:port` for distributed setups. |
 
 ### Server Configuration
@@ -256,6 +258,12 @@ Once enabled, configure lifecycle rules via:
 | `MULTIPART_MIN_PART_SIZE` | `5242880` (5 MB) | Minimum part size for multipart uploads. |
 | `BUCKET_STATS_CACHE_TTL` | `60` | Seconds to cache bucket statistics. |
 | `BULK_DELETE_MAX_KEYS` | `500` | Maximum keys per bulk delete request. |
+| `BULK_DOWNLOAD_MAX_BYTES` | `1073741824` (1 GiB) | Maximum total size for bulk ZIP downloads. |
+| `OBJECT_CACHE_TTL` | `60` | Seconds to cache object metadata. |
+
+#### Gzip Compression
+
+API responses for JSON, XML, HTML, CSS, and JavaScript are automatically gzip-compressed when the client sends `Accept-Encoding: gzip`. Compression activates for responses larger than 500 bytes and is handled by a WSGI middleware (`app/compression.py`). Binary object downloads and streaming responses are never compressed. No configuration is needed.
 
 ### Server Settings
 
@@ -285,6 +293,12 @@ If running behind a reverse proxy (e.g., Nginx, Cloudflare, or a tunnel), ensure
 
 The application automatically trusts these headers to generate correct presigned URLs (e.g., `https://s3.example.com/...` instead of `http://127.0.0.1:5000/...`). Alternatively, you can explicitly set `API_BASE_URL` to your public endpoint.
 
+| Variable | Default | Notes |
+| --- | --- | --- |
+| `NUM_TRUSTED_PROXIES` | `1` | Number of trusted reverse proxies for `X-Forwarded-*` header processing. |
+| `ALLOWED_REDIRECT_HOSTS` | `""` | Comma-separated whitelist of safe redirect targets. Empty allows only same-host redirects. |
+| `ALLOW_INTERNAL_ENDPOINTS` | `false` | Allow connections to internal/private IPs for webhooks and replication targets. **Keep disabled in production unless needed.** |
+
 ## 4. Upgrading and Updates
 
 ### Version Checking
@@ -912,7 +926,7 @@ Objects with forward slashes (`/`) in their keys are displayed as a folder hiera
 
 - Select multiple objects using checkboxes
 - **Bulk Delete**: Delete multiple objects at once
-- **Bulk Download**: Download selected objects as individual files
+- **Bulk Download**: Download selected objects as a single ZIP archive (up to `BULK_DOWNLOAD_MAX_BYTES`, default 1 GiB)
 
 #### Search & Filter
 
@@ -985,6 +999,7 @@ MyFSIO supports **server-side encryption at rest** to protect your data. When en
 |------|-------------|
 | **AES-256 (SSE-S3)** | Server-managed encryption using a local master key |
 | **KMS (SSE-KMS)** | Encryption using customer-managed keys via the built-in KMS |
+| **SSE-C** | Server-side encryption with customer-provided keys (per-request) |
 
 ### Enabling Encryption
 
@@ -1083,6 +1098,44 @@ encrypted, metadata = ClientEncryptionHelper.encrypt_for_upload(plaintext, key)
 decrypted = ClientEncryptionHelper.decrypt_from_download(encrypted, metadata, key)
 ```
 
+### SSE-C (Customer-Provided Keys)
+
+With SSE-C, you provide your own 256-bit AES encryption key with each request. The server encrypts/decrypts using your key but never stores it. You must supply the same key for both upload and download.
+
+**Required headers:**
+
+| Header | Value |
+|--------|-------|
+| `x-amz-server-side-encryption-customer-algorithm` | `AES256` |
+| `x-amz-server-side-encryption-customer-key` | Base64-encoded 256-bit key |
+| `x-amz-server-side-encryption-customer-key-MD5` | Base64-encoded MD5 of the key |
+
+```bash
+# Generate a 256-bit key
+KEY=$(openssl rand -base64 32)
+KEY_MD5=$(echo -n "$KEY" | base64 -d | openssl dgst -md5 -binary | base64)
+
+# Upload with SSE-C
+curl -X PUT "http://localhost:5000/my-bucket/secret.txt" \
+  -H "X-Access-Key: ..." -H "X-Secret-Key: ..." \
+  -H "x-amz-server-side-encryption-customer-algorithm: AES256" \
+  -H "x-amz-server-side-encryption-customer-key: $KEY" \
+  -H "x-amz-server-side-encryption-customer-key-MD5: $KEY_MD5" \
+  --data-binary @secret.txt
+
+# Download with SSE-C (same key required)
+curl "http://localhost:5000/my-bucket/secret.txt" \
+  -H "X-Access-Key: ..." -H "X-Secret-Key: ..." \
+  -H "x-amz-server-side-encryption-customer-algorithm: AES256" \
+  -H "x-amz-server-side-encryption-customer-key: $KEY" \
+  -H "x-amz-server-side-encryption-customer-key-MD5: $KEY_MD5"
+```
+
+**Key points:**
+- SSE-C does not require `ENCRYPTION_ENABLED` or `KMS_ENABLED` — the key is provided per-request
+- If you lose your key, the data is irrecoverable
+- The MD5 header is optional but recommended for integrity verification
+
 ### Important Notes
 
 - **Existing objects are NOT encrypted** - Only new uploads after enabling encryption are encrypted
@@ -1959,6 +2012,20 @@ curl -X PUT "http://localhost:5000/my-bucket/file.txt" \
   -H "x-amz-meta-newkey: newvalue"
 ```
 
+### MoveObject (UI)
+
+Move an object to a different key or bucket. This is a UI-only convenience operation that performs a copy followed by a delete of the source. Requires `read` and `delete` on the source, and `write` on the destination.
+
+```bash
+# Move via UI API
+curl -X POST "http://localhost:5100/ui/buckets/my-bucket/objects/old-path/file.txt/move" \
+  -H "Content-Type: application/json" \
+  --cookie "session=..." \
+  -d '{"dest_bucket": "other-bucket", "dest_key": "new-path/file.txt"}'
+```
+
+The move is atomic from the caller's perspective: if the copy succeeds but the delete fails, the object exists in both locations (no data loss).
+
 ### UploadPartCopy
 
 Copy data from an existing object into a multipart upload part:

myfsio_core/Cargo.lock

@@ -1,421 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 4
-
-[[package]]
-name = "aho-corasick"
-version = "1.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "allocator-api2"
-version = "0.2.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
-
-[[package]]
-name = "bitflags"
-version = "2.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
-
-[[package]]
-name = "block-buffer"
-version = "0.10.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
-dependencies = [
- "generic-array",
-]
-
-[[package]]
-name = "cfg-if"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
-
-[[package]]
-name = "cpufeatures"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "crypto-common"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
-dependencies = [
- "generic-array",
- "typenum",
-]
-
-[[package]]
-name = "digest"
-version = "0.10.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
-dependencies = [
- "block-buffer",
- "crypto-common",
- "subtle",
-]
-
-[[package]]
-name = "equivalent"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
-
-[[package]]
-name = "foldhash"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
-
-[[package]]
-name = "generic-array"
-version = "0.14.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
-dependencies = [
- "typenum",
- "version_check",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.15.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
-dependencies = [
- "allocator-api2",
- "equivalent",
- "foldhash",
-]
-
-[[package]]
-name = "heck"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
-
-[[package]]
-name = "hex"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
-
-[[package]]
-name = "hmac"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
-dependencies = [
- "digest",
-]
-
-[[package]]
-name = "libc"
-version = "0.2.182"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
-
-[[package]]
-name = "lock_api"
-version = "0.4.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
-dependencies = [
- "scopeguard",
-]
-
-[[package]]
-name = "lru"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f8cc7106155f10bdf99a6f379688f543ad6596a415375b36a59a054ceda1198"
-dependencies = [
- "hashbrown",
-]
-
-[[package]]
-name = "md-5"
-version = "0.10.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
-dependencies = [
- "cfg-if",
- "digest",
-]
-
-[[package]]
-name = "memchr"
-version = "2.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
-
-[[package]]
-name = "myfsio_core"
-version = "0.1.0"
-dependencies = [
- "hex",
- "hmac",
- "lru",
- "md-5",
- "parking_lot",
- "pyo3",
- "regex",
- "sha2",
- "unicode-normalization",
-]
-
-[[package]]
-name = "once_cell"
-version = "1.21.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
-
-[[package]]
-name = "parking_lot"
-version = "0.12.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
-dependencies = [
- "lock_api",
- "parking_lot_core",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.9.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
-dependencies = [
- "cfg-if",
- "libc",
- "redox_syscall",
- "smallvec",
- "windows-link",
-]
-
-[[package]]
-name = "portable-atomic"
-version = "1.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.106"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "pyo3"
-version = "0.28.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14c738662e2181be11cb82487628404254902bb3225d8e9e99c31f3ef82a405c"
-dependencies = [
- "libc",
- "once_cell",
- "portable-atomic",
- "pyo3-build-config",
- "pyo3-ffi",
- "pyo3-macros",
-]
-
-[[package]]
-name = "pyo3-build-config"
-version = "0.28.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9ca0864a7dd3c133a7f3f020cbff2e12e88420da854c35540fd20ce2d60e435"
-dependencies = [
- "target-lexicon",
-]
-
-[[package]]
-name = "pyo3-ffi"
-version = "0.28.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9dfc1956b709823164763a34cc42bbfd26b8730afa77809a3df8b94a3ae3b059"
-dependencies = [
- "libc",
- "pyo3-build-config",
-]
-
-[[package]]
-name = "pyo3-macros"
-version = "0.28.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29dc660ad948bae134d579661d08033fbb1918f4529c3bbe3257a68f2009ddf2"
-dependencies = [
- "proc-macro2",
- "pyo3-macros-backend",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "pyo3-macros-backend"
-version = "0.28.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e78cd6c6d718acfcedf26c3d21fe0f053624368b0d44298c55d7138fde9331f7"
-dependencies = [
- "heck",
- "proc-macro2",
- "pyo3-build-config",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.44"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.5.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
-dependencies = [
- "bitflags",
-]
-
-[[package]]
-name = "regex"
-version = "1.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-automata",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.4.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-syntax"
-version = "0.8.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c"
-
-[[package]]
-name = "scopeguard"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
-
-[[package]]
-name = "sha2"
-version = "0.10.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
-dependencies = [
- "cfg-if",
- "cpufeatures",
- "digest",
-]
-
-[[package]]
-name = "smallvec"
-version = "1.15.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
-
-[[package]]
-name = "subtle"
-version = "2.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
-
-[[package]]
-name = "syn"
-version = "2.0.116"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3df424c70518695237746f84cede799c9c58fcb37450d7b23716568cc8bc69cb"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "target-lexicon"
-version = "0.13.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
-
-[[package]]
-name = "tinyvec"
-version = "1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa"
-dependencies = [
- "tinyvec_macros",
-]
-
-[[package]]
-name = "tinyvec_macros"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
-
-[[package]]
-name = "typenum"
-version = "1.19.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
-
-[[package]]
-name = "unicode-normalization"
-version = "0.1.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
-dependencies = [
- "tinyvec",
-]
-
-[[package]]
-name = "version_check"
-version = "0.9.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
-
-[[package]]
-name = "windows-link"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"

myfsio_core/Cargo.toml

@@ -14,6 +14,8 @@ sha2 = "0.10"
 md-5 = "0.10"
 hex = "0.4"
 unicode-normalization = "0.1"
+serde_json = "1"
 regex = "1"
 lru = "0.14"
 parking_lot = "0.12"
+percent-encoding = "2"

myfsio_core/src/lib.rs

@@ -1,4 +1,5 @@
 mod hashing;
+mod metadata;
 mod sigv4;
 mod validation;
 
@@ -10,6 +11,7 @@ mod myfsio_core {
 
     #[pymodule_init]
     fn init(m: &Bound<'_, PyModule>) -> PyResult<()> {
+        m.add_function(wrap_pyfunction!(sigv4::verify_sigv4_signature, m)?)?;
         m.add_function(wrap_pyfunction!(sigv4::derive_signing_key, m)?)?;
         m.add_function(wrap_pyfunction!(sigv4::compute_signature, m)?)?;
         m.add_function(wrap_pyfunction!(sigv4::build_string_to_sign, m)?)?;
@@ -25,6 +27,8 @@ mod myfsio_core {
         m.add_function(wrap_pyfunction!(validation::validate_object_key, m)?)?;
         m.add_function(wrap_pyfunction!(validation::validate_bucket_name, m)?)?;
 
+        m.add_function(wrap_pyfunction!(metadata::read_index_entry, m)?)?;
+
         Ok(())
     }
 }

myfsio_core/src/metadata.rs

@@ -0,0 +1,71 @@
+use pyo3::exceptions::PyValueError;
+use pyo3::prelude::*;
+use pyo3::types::{PyDict, PyList, PyString};
+use serde_json::Value;
+use std::fs;
+
+const MAX_DEPTH: u32 = 64;
+
+fn value_to_py(py: Python<'_>, v: &Value, depth: u32) -> PyResult<Py<PyAny>> {
+    if depth > MAX_DEPTH {
+        return Err(PyValueError::new_err("JSON nesting too deep"));
+    }
+    match v {
+        Value::Null => Ok(py.None()),
+        Value::Bool(b) => Ok((*b).into_pyobject(py)?.to_owned().into_any().unbind()),
+        Value::Number(n) => {
+            if let Some(i) = n.as_i64() {
+                Ok(i.into_pyobject(py)?.into_any().unbind())
+            } else if let Some(f) = n.as_f64() {
+                Ok(f.into_pyobject(py)?.into_any().unbind())
+            } else {
+                Ok(py.None())
+            }
+        }
+        Value::String(s) => Ok(PyString::new(py, s).into_any().unbind()),
+        Value::Array(arr) => {
+            let list = PyList::empty(py);
+            for item in arr {
+                list.append(value_to_py(py, item, depth + 1)?)?;
+            }
+            Ok(list.into_any().unbind())
+        }
+        Value::Object(map) => {
+            let dict = PyDict::new(py);
+            for (k, val) in map {
+                dict.set_item(k, value_to_py(py, val, depth + 1)?)?;
+            }
+            Ok(dict.into_any().unbind())
+        }
+    }
+}
+
+#[pyfunction]
+pub fn read_index_entry(
+    py: Python<'_>,
+    path: &str,
+    entry_name: &str,
+) -> PyResult<Option<Py<PyAny>>> {
+    let path_owned = path.to_owned();
+    let entry_owned = entry_name.to_owned();
+
+    let entry: Option<Value> = py.detach(move || -> PyResult<Option<Value>> {
+        let content = match fs::read_to_string(&path_owned) {
+            Ok(c) => c,
+            Err(_) => return Ok(None),
+        };
+        let parsed: Value = match serde_json::from_str(&content) {
+            Ok(v) => v,
+            Err(_) => return Ok(None),
+        };
+        match parsed {
+            Value::Object(mut map) => Ok(map.remove(&entry_owned)),
+            _ => Ok(None),
+        }
+    })?;
+
+    match entry {
+        Some(val) => Ok(Some(value_to_py(py, &val, 0)?)),
+        None => Ok(None),
+    }
+}

myfsio_core/src/sigv4.rs

@@ -1,6 +1,7 @@
 use hmac::{Hmac, Mac};
 use lru::LruCache;
 use parking_lot::Mutex;
+use percent_encoding::{percent_encode, AsciiSet, NON_ALPHANUMERIC};
 use pyo3::prelude::*;
 use sha2::{Digest, Sha256};
 use std::num::NonZeroUsize;
@@ -19,14 +20,29 @@ static SIGNING_KEY_CACHE: LazyLock<Mutex<LruCache<(String, String, String, Strin
 
 const CACHE_TTL_SECS: u64 = 60;
 
+const AWS_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC
+    .remove(b'-')
+    .remove(b'_')
+    .remove(b'.')
+    .remove(b'~');
+
 fn hmac_sha256(key: &[u8], msg: &[u8]) -> Vec<u8> {
     let mut mac = HmacSha256::new_from_slice(key).expect("HMAC key length is always valid");
     mac.update(msg);
     mac.finalize().into_bytes().to_vec()
 }
 
-#[pyfunction]
-pub fn derive_signing_key(
+fn sha256_hex(data: &[u8]) -> String {
+    let mut hasher = Sha256::new();
+    hasher.update(data);
+    hex::encode(hasher.finalize())
+}
+
+fn aws_uri_encode(input: &str) -> String {
+    percent_encode(input.as_bytes(), AWS_ENCODE_SET).to_string()
+}
+
+fn derive_signing_key_cached(
     secret_key: &str,
     date_stamp: &str,
     region: &str,
@@ -68,18 +84,91 @@ pub fn derive_signing_key(
     k_signing
 }
 
+fn constant_time_compare_inner(a: &[u8], b: &[u8]) -> bool {
+    if a.len() != b.len() {
+        return false;
+    }
+    let mut result: u8 = 0;
+    for (x, y) in a.iter().zip(b.iter()) {
+        result |= x ^ y;
+    }
+    result == 0
+}
+
+#[pyfunction]
+pub fn verify_sigv4_signature(
+    method: &str,
+    canonical_uri: &str,
+    query_params: Vec<(String, String)>,
+    signed_headers_str: &str,
+    header_values: Vec<(String, String)>,
+    payload_hash: &str,
+    amz_date: &str,
+    date_stamp: &str,
+    region: &str,
+    service: &str,
+    secret_key: &str,
+    provided_signature: &str,
+) -> bool {
+    let mut sorted_params = query_params;
+    sorted_params.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)));
+
+    let canonical_query_string = sorted_params
+        .iter()
+        .map(|(k, v)| format!("{}={}", aws_uri_encode(k), aws_uri_encode(v)))
+        .collect::<Vec<_>>()
+        .join("&");
+
+    let mut canonical_headers = String::new();
+    for (name, value) in &header_values {
+        let lower_name = name.to_lowercase();
+        let normalized = value.split_whitespace().collect::<Vec<_>>().join(" ");
+        let final_value = if lower_name == "expect" && normalized.is_empty() {
+            "100-continue"
+        } else {
+            &normalized
+        };
+        canonical_headers.push_str(&lower_name);
+        canonical_headers.push(':');
+        canonical_headers.push_str(final_value);
+        canonical_headers.push('\n');
+    }
+
+    let canonical_request = format!(
+        "{}\n{}\n{}\n{}\n{}\n{}",
+        method, canonical_uri, canonical_query_string, canonical_headers, signed_headers_str, payload_hash
+    );
+
+    let credential_scope = format!("{}/{}/{}/aws4_request", date_stamp, region, service);
+    let cr_hash = sha256_hex(canonical_request.as_bytes());
+    let string_to_sign = format!(
+        "AWS4-HMAC-SHA256\n{}\n{}\n{}",
+        amz_date, credential_scope, cr_hash
+    );
+
+    let signing_key = derive_signing_key_cached(secret_key, date_stamp, region, service);
+    let calculated = hmac_sha256(&signing_key, string_to_sign.as_bytes());
+    let calculated_hex = hex::encode(&calculated);
+
+    constant_time_compare_inner(calculated_hex.as_bytes(), provided_signature.as_bytes())
+}
+
+#[pyfunction]
+pub fn derive_signing_key(
+    secret_key: &str,
+    date_stamp: &str,
+    region: &str,
+    service: &str,
+) -> Vec<u8> {
+    derive_signing_key_cached(secret_key, date_stamp, region, service)
+}
+
 #[pyfunction]
 pub fn compute_signature(signing_key: &[u8], string_to_sign: &str) -> String {
     let sig = hmac_sha256(signing_key, string_to_sign.as_bytes());
     hex::encode(sig)
 }
 
-fn sha256_hex(data: &[u8]) -> String {
-    let mut hasher = Sha256::new();
-    hasher.update(data);
-    hex::encode(hasher.finalize())
-}
-
 #[pyfunction]
 pub fn build_string_to_sign(
     amz_date: &str,
@@ -87,19 +176,15 @@ pub fn build_string_to_sign(
     canonical_request: &str,
 ) -> String {
     let cr_hash = sha256_hex(canonical_request.as_bytes());
-    format!("AWS4-HMAC-SHA256\n{}\n{}\n{}", amz_date, credential_scope, cr_hash)
+    format!(
+        "AWS4-HMAC-SHA256\n{}\n{}\n{}",
+        amz_date, credential_scope, cr_hash
+    )
 }
 
 #[pyfunction]
 pub fn constant_time_compare(a: &str, b: &str) -> bool {
-    if a.len() != b.len() {
-        return false;
-    }
-    let mut result: u8 = 0;
-    for (x, y) in a.bytes().zip(b.bytes()) {
-        result |= x ^ y;
-    }
-    result == 0
+    constant_time_compare_inner(a.as_bytes(), b.as_bytes())
 }
 
 #[pyfunction]

myfsio_core/target/.rustc_info.json

@@ -1 +0,0 @@
-{"rustc_fingerprint":13172970000770725120,"outputs":{"7971740275564407648":{"success":true,"status":"","code":0,"stdout":"___.exe\nlib___.rlib\n___.dll\n___.dll\n___.lib\n___.dll\nC:\\Users\\jun\\.rustup\\toolchains\\stable-x86_64-pc-windows-msvc\npacked\n___\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_abi=\"\"\ntarget_arch=\"x86_64\"\ntarget_endian=\"little\"\ntarget_env=\"msvc\"\ntarget_family=\"windows\"\ntarget_feature=\"cmpxchg16b\"\ntarget_feature=\"fxsr\"\ntarget_feature=\"sse\"\ntarget_feature=\"sse2\"\ntarget_feature=\"sse3\"\ntarget_has_atomic=\"128\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_os=\"windows\"\ntarget_pointer_width=\"64\"\ntarget_vendor=\"pc\"\nwindows\n","stderr":""},"17747080675513052775":{"success":true,"status":"","code":0,"stdout":"rustc 1.93.1 (01f6ddf75 2026-02-11)\nbinary: rustc\ncommit-hash: 01f6ddf7588f42ae2d7eb0a2f21d44e8e96674cf\ncommit-date: 2026-02-11\nhost: x86_64-pc-windows-msvc\nrelease: 1.93.1\nLLVM version: 21.1.8\n","stderr":""}},"successes":{}}

myfsio_core/target/CACHEDIR.TAG

@@ -1,3 +0,0 @@
-Signature: 8a477f597d28d172789f06886806bc55
-# This file is a cache directory tag created by cargo.
-# For information about cache directory tags see https://bford.info/cachedir/

myfsio_core/target/release/.fingerprint/aho-corasick-45694771b543be75/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/aho-corasick-45694771b543be75/lib-aho_corasick

@@ -1 +0,0 @@
-801af22cf202da8e

myfsio_core/target/release/.fingerprint/aho-corasick-45694771b543be75/lib-aho_corasick.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"perf-literal\", \"std\"]","declared_features":"[\"default\", \"logging\", \"perf-literal\", \"std\"]","target":7534583537114156500,"profile":2040997289075261528,"path":6364296192483896971,"deps":[[1363051979936526615,"memchr",false,11090220145123168660]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\aho-corasick-45694771b543be75\\dep-lib-aho_corasick","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/allocator-api2-db7934dbe96de5b4/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/allocator-api2-db7934dbe96de5b4/lib-allocator_api2

@@ -1 +0,0 @@
-435555ec2fb592e3

myfsio_core/target/release/.fingerprint/allocator-api2-db7934dbe96de5b4/lib-allocator_api2.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"alloc\"]","declared_features":"[\"alloc\", \"default\", \"fresh-rust\", \"nightly\", \"serde\", \"std\"]","target":5388200169723499962,"profile":4067574213046180398,"path":10654049299693593327,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\allocator-api2-db7934dbe96de5b4\\dep-lib-allocator_api2","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/autocfg-1c4fb7a37cc3df69/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/autocfg-1c4fb7a37cc3df69/lib-autocfg

@@ -1 +0,0 @@
-d28af275d001c358

myfsio_core/target/release/.fingerprint/autocfg-1c4fb7a37cc3df69/lib-autocfg.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":6962977057026645649,"profile":1369601567987815722,"path":9853093265219907461,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\autocfg-1c4fb7a37cc3df69\\dep-lib-autocfg","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/block-buffer-95b0ac364bec72f9/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/block-buffer-95b0ac364bec72f9/lib-block_buffer

@@ -1 +0,0 @@
-1fbf4ba9542edced

myfsio_core/target/release/.fingerprint/block-buffer-95b0ac364bec72f9/lib-block_buffer.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":4098124618827574291,"profile":2040997289075261528,"path":3658007358608479489,"deps":[[10520923840501062997,"generic_array",false,11555283918993371487]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\block-buffer-95b0ac364bec72f9\\dep-lib-block_buffer","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/cfg-if-be2711f84a777e73/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/cfg-if-be2711f84a777e73/lib-cfg_if

@@ -1 +0,0 @@
-37923e6f5f9687ab

myfsio_core/target/release/.fingerprint/cfg-if-be2711f84a777e73/lib-cfg_if.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[\"core\", \"rustc-dep-of-std\"]","target":13840298032947503755,"profile":2040997289075261528,"path":4093486168504982869,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\cfg-if-be2711f84a777e73\\dep-lib-cfg_if","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/cpufeatures-980094f8735c42d1/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/cpufeatures-980094f8735c42d1/lib-cpufeatures

@@ -1 +0,0 @@
-603e28136cf5763c

myfsio_core/target/release/.fingerprint/cpufeatures-980094f8735c42d1/lib-cpufeatures.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":2330704043955282025,"profile":2040997289075261528,"path":13200428550696548327,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\cpufeatures-980094f8735c42d1\\dep-lib-cpufeatures","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/crypto-common-289a508abdda3048/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/crypto-common-289a508abdda3048/lib-crypto_common

@@ -1 +0,0 @@
-896672d759b5299c

myfsio_core/target/release/.fingerprint/crypto-common-289a508abdda3048/lib-crypto_common.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"std\"]","declared_features":"[\"getrandom\", \"rand_core\", \"std\"]","target":12082577455412410174,"profile":2040997289075261528,"path":14902376638882023040,"deps":[[857979250431893282,"typenum",false,7416411392359930020],[10520923840501062997,"generic_array",false,11555283918993371487]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\crypto-common-289a508abdda3048\\dep-lib-crypto_common","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/digest-a91458bfa5613332/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/digest-a91458bfa5613332/lib-digest

@@ -1 +0,0 @@
-914a617b9f05c9d8

myfsio_core/target/release/.fingerprint/digest-a91458bfa5613332/lib-digest.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"alloc\", \"block-buffer\", \"core-api\", \"default\", \"mac\", \"std\", \"subtle\"]","declared_features":"[\"alloc\", \"blobby\", \"block-buffer\", \"const-oid\", \"core-api\", \"default\", \"dev\", \"mac\", \"oid\", \"rand_core\", \"std\", \"subtle\"]","target":7510122432137863311,"profile":2040997289075261528,"path":11503432597517024930,"deps":[[6039282458970808711,"crypto_common",false,11252724541433210505],[10626340395483396037,"block_buffer",false,17139625223017709343],[17003143334332120809,"subtle",false,8597342066671925934]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\digest-a91458bfa5613332\\dep-lib-digest","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/equivalent-943ac856871c0988/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/equivalent-943ac856871c0988/lib-equivalent

@@ -1 +0,0 @@
-3b95cf48bbd7dc53

myfsio_core/target/release/.fingerprint/equivalent-943ac856871c0988/lib-equivalent.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":1524667692659508025,"profile":2040997289075261528,"path":17534356223679657546,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\equivalent-943ac856871c0988\\dep-lib-equivalent","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/foldhash-b8a92f8c10d550f7/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/foldhash-b8a92f8c10d550f7/lib-foldhash

@@ -1 +0,0 @@
-b7ba5182ce570398

myfsio_core/target/release/.fingerprint/foldhash-b8a92f8c10d550f7/lib-foldhash.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[\"default\", \"std\"]","target":18077926938045032029,"profile":2040997289075261528,"path":9869209539952544870,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\foldhash-b8a92f8c10d550f7\\dep-lib-foldhash","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/generic-array-2462daa120fe5936/build-script-build-script-build

@@ -1 +0,0 @@
-f0a5af4d8a8c7106

myfsio_core/target/release/.fingerprint/generic-array-2462daa120fe5936/build-script-build-script-build.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"more_lengths\"]","declared_features":"[\"more_lengths\", \"serde\", \"zeroize\"]","target":12318548087768197662,"profile":1369601567987815722,"path":13853454403963664247,"deps":[[5398981501050481332,"version_check",false,16419025953046340415]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\generic-array-2462daa120fe5936\\dep-build-script-build-script-build","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/generic-array-2462daa120fe5936/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/generic-array-62216349963f3a3c/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/generic-array-62216349963f3a3c/lib-generic_array

@@ -1 +0,0 @@
-5f316276809d5ca0

myfsio_core/target/release/.fingerprint/generic-array-62216349963f3a3c/lib-generic_array.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"more_lengths\"]","declared_features":"[\"more_lengths\", \"serde\", \"zeroize\"]","target":13084005262763373425,"profile":2040997289075261528,"path":12463275850883329568,"deps":[[857979250431893282,"typenum",false,7416411392359930020],[10520923840501062997,"build_script_build",false,16977603856295925732]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\generic-array-62216349963f3a3c\\dep-lib-generic_array","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/generic-array-a1f36f10f654fafd/run-build-script-build-script-build

@@ -1 +0,0 @@
-e417d28fc1909ceb

myfsio_core/target/release/.fingerprint/generic-array-a1f36f10f654fafd/run-build-script-build-script-build.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"","declared_features":"","target":0,"profile":0,"path":0,"deps":[[10520923840501062997,"build_script_build",false,464306762232604144]],"local":[{"Precalculated":"0.14.7"}],"rustflags":[],"config":0,"compile_kind":0}

myfsio_core/target/release/.fingerprint/hashbrown-510d641b592c306b/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/hashbrown-510d641b592c306b/lib-hashbrown

@@ -1 +0,0 @@
-aec88a641c5288e3

myfsio_core/target/release/.fingerprint/hashbrown-510d641b592c306b/lib-hashbrown.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"allocator-api2\", \"default\", \"default-hasher\", \"equivalent\", \"inline-more\", \"raw-entry\"]","declared_features":"[\"alloc\", \"allocator-api2\", \"core\", \"default\", \"default-hasher\", \"equivalent\", \"inline-more\", \"nightly\", \"raw-entry\", \"rayon\", \"rustc-dep-of-std\", \"rustc-internal-api\", \"serde\"]","target":13796197676120832388,"profile":2040997289075261528,"path":12448322139402656924,"deps":[[5230392855116717286,"equivalent",false,6042941999404782907],[9150530836556604396,"allocator_api2",false,16398368410642502979],[10842263908529601448,"foldhash",false,10953695263156452023]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\hashbrown-510d641b592c306b\\dep-lib-hashbrown","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/heck-b47c94fd2a7e00cb/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/heck-b47c94fd2a7e00cb/lib-heck

@@ -1 +0,0 @@
-ddc0b590ff80762b

myfsio_core/target/release/.fingerprint/heck-b47c94fd2a7e00cb/lib-heck.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":17886154901722686619,"profile":1369601567987815722,"path":8608102977929876445,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\heck-b47c94fd2a7e00cb\\dep-lib-heck","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/hex-253414d2260adcdf/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/hex-253414d2260adcdf/lib-hex

@@ -1 +0,0 @@
-41890ebff4143fa5

myfsio_core/target/release/.fingerprint/hex-253414d2260adcdf/lib-hex.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"alloc\", \"default\", \"std\"]","declared_features":"[\"alloc\", \"default\", \"serde\", \"std\"]","target":4242469766639956503,"profile":2040997289075261528,"path":6793865871540733919,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\hex-253414d2260adcdf\\dep-lib-hex","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/hmac-3297e61b9effb758/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/hmac-3297e61b9effb758/lib-hmac

@@ -1 +0,0 @@
-3f45b8d062d94ba4

myfsio_core/target/release/.fingerprint/hmac-3297e61b9effb758/lib-hmac.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[\"reset\", \"std\"]","target":12991177224612424488,"profile":2040997289075261528,"path":17893893568771568113,"deps":[[17475753849556516473,"digest",false,15621022965039188625]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\hmac-3297e61b9effb758\\dep-lib-hmac","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/indoc-0c686c3f403a2566/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/indoc-0c686c3f403a2566/lib-indoc

@@ -1 +0,0 @@
-9896adc8892b3fe4

myfsio_core/target/release/.fingerprint/indoc-0c686c3f403a2566/lib-indoc.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":8726396592336845528,"profile":1369601567987815722,"path":18304219166357541938,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\indoc-0c686c3f403a2566\\dep-lib-indoc","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/libc-5f9f280eeaad3bb3/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/libc-5f9f280eeaad3bb3/lib-libc

@@ -1 +0,0 @@
-3c0b7985f088ae56

myfsio_core/target/release/.fingerprint/libc-5f9f280eeaad3bb3/lib-libc.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"default\", \"std\"]","declared_features":"[\"align\", \"const-extern-fn\", \"default\", \"extra_traits\", \"rustc-dep-of-std\", \"rustc-std-workspace-core\", \"std\", \"use_std\"]","target":17682796336736096309,"profile":7322064999780386650,"path":3108645287704295931,"deps":[[18365559012052052344,"build_script_build",false,12197584826291254217]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\libc-5f9f280eeaad3bb3\\dep-lib-libc","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/libc-70cd639287284bb6/build-script-build-script-build

@@ -1 +0,0 @@
-7e48e2b3ad86c125

myfsio_core/target/release/.fingerprint/libc-70cd639287284bb6/build-script-build-script-build.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"default\", \"std\"]","declared_features":"[\"align\", \"const-extern-fn\", \"default\", \"extra_traits\", \"rustc-dep-of-std\", \"rustc-std-workspace-core\", \"std\", \"use_std\"]","target":5408242616063297496,"profile":8928907579149787682,"path":7198683120865577851,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\libc-70cd639287284bb6\\dep-build-script-build-script-build","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/libc-70cd639287284bb6/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.