Merge pull request 'MyFSIO v0.3.4 Release' (#27) from next into main

Reviewed-on: #27

.gitignore

@@ -26,6 +26,10 @@ dist/
 *.egg-info/
 .eggs/
 
+# Rust / maturin build artifacts
+myfsio_core/target/
+myfsio_core/Cargo.lock
+
 # Local runtime artifacts
 logs/
 *.log

README.md

@@ -80,7 +80,7 @@ python run.py --mode api   # API only (port 5000)
 python run.py --mode ui    # UI only (port 5100)
 ```
 
-**Default Credentials:** `localadmin` / `localadmin`
+**Credentials:** Generated automatically on first run and printed to the console. If missed, check the IAM config file at `<STORAGE_ROOT>/.myfsio.sys/config/iam.json`.
 
 - **Web Console:** http://127.0.0.1:5100/ui
 - **API Endpoint:** http://127.0.0.1:5000

app/__init__.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import html as html_module
 import logging
 import mimetypes
+import os
 import shutil
 import sys
 import time
@@ -93,7 +94,14 @@ def create_app(
         app.config.setdefault("WTF_CSRF_ENABLED", False)
 
     # Trust X-Forwarded-* headers from proxies
-    app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1)
+    num_proxies = app.config.get("NUM_TRUSTED_PROXIES", 1)
+    if num_proxies:
+        if "NUM_TRUSTED_PROXIES" not in os.environ:
+            logging.getLogger(__name__).warning(
+                "NUM_TRUSTED_PROXIES not set, defaulting to 1. "
+                "Set NUM_TRUSTED_PROXIES=0 if not behind a reverse proxy."
+            )
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_for=num_proxies, x_proto=num_proxies, x_host=num_proxies, x_prefix=num_proxies)
 
     # Enable gzip compression for responses (10-20x smaller JSON payloads)
     if app.config.get("ENABLE_GZIP", True):
@@ -107,7 +115,7 @@ def create_app(
 
     storage = ObjectStorage(
         Path(app.config["STORAGE_ROOT"]),
-        cache_ttl=app.config.get("OBJECT_CACHE_TTL", 5),
+        cache_ttl=app.config.get("OBJECT_CACHE_TTL", 60),
         object_cache_max_size=app.config.get("OBJECT_CACHE_MAX_SIZE", 100),
         bucket_config_cache_ttl=app.config.get("BUCKET_CONFIG_CACHE_TTL_SECONDS", 30.0),
         object_key_max_length_bytes=app.config.get("OBJECT_KEY_MAX_LENGTH_BYTES", 1024),

app/bucket_policies.py

@@ -75,7 +75,7 @@ def _evaluate_condition_operator(
         expected_null = condition_values[0].lower() in ("true", "1", "yes") if condition_values else True
         return is_null == expected_null
 
-    return True
+    return False
 
 ACTION_ALIASES = {
     "s3:listbucket": "list",

app/config.py

@@ -241,7 +241,7 @@ class AppConfig:
         cors_expose_headers = _csv(str(_get("CORS_EXPOSE_HEADERS", "*")), ["*"])
         session_lifetime_days = int(_get("SESSION_LIFETIME_DAYS", 30))
         bucket_stats_cache_ttl = int(_get("BUCKET_STATS_CACHE_TTL", 60))
-        object_cache_ttl = int(_get("OBJECT_CACHE_TTL", 5))
+        object_cache_ttl = int(_get("OBJECT_CACHE_TTL", 60))
 
         encryption_enabled = str(_get("ENCRYPTION_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
         encryption_keys_dir = storage_root / ".myfsio.sys" / "keys"
@@ -314,7 +314,7 @@ class AppConfig:
         site_region = str(_get("SITE_REGION", "us-east-1"))
         site_priority = int(_get("SITE_PRIORITY", 100))
         ratelimit_admin = _validate_rate_limit(str(_get("RATE_LIMIT_ADMIN", "60 per minute")))
-        num_trusted_proxies = int(_get("NUM_TRUSTED_PROXIES", 0))
+        num_trusted_proxies = int(_get("NUM_TRUSTED_PROXIES", 1))
         allowed_redirect_hosts_raw = _get("ALLOWED_REDIRECT_HOSTS", "")
         allowed_redirect_hosts = [h.strip() for h in str(allowed_redirect_hosts_raw).split(",") if h.strip()]
         allow_internal_endpoints = str(_get("ALLOW_INTERNAL_ENDPOINTS", "0")).lower() in {"1", "true", "yes", "on"}

app/encrypted_storage.py

@@ -189,7 +189,13 @@ class EncryptedObjectStorage:
     
     def list_objects(self, bucket_name: str, **kwargs):
         return self.storage.list_objects(bucket_name, **kwargs)
-    
+
+    def list_objects_shallow(self, bucket_name: str, **kwargs):
+        return self.storage.list_objects_shallow(bucket_name, **kwargs)
+
+    def search_objects(self, bucket_name: str, query: str, **kwargs):
+        return self.storage.search_objects(bucket_name, query, **kwargs)
+
     def list_objects_all(self, bucket_name: str):
         return self.storage.list_objects_all(bucket_name)
     

app/encryption.py

@@ -19,6 +19,13 @@ from cryptography.hazmat.primitives import hashes
 if sys.platform != "win32":
     import fcntl
 
+try:
+    import myfsio_core as _rc
+    _HAS_RUST = True
+except ImportError:
+    _rc = None
+    _HAS_RUST = False
+
 logger = logging.getLogger(__name__)
 
 
@@ -338,6 +345,69 @@ class StreamingEncryptor:
         output.seek(0)
         return output
 
+    def encrypt_file(self, input_path: str, output_path: str) -> EncryptionMetadata:
+        data_key, encrypted_data_key = self.provider.generate_data_key()
+        base_nonce = secrets.token_bytes(12)
+
+        if _HAS_RUST:
+            _rc.encrypt_stream_chunked(
+                input_path, output_path, data_key, base_nonce, self.chunk_size
+            )
+        else:
+            with open(input_path, "rb") as stream:
+                aesgcm = AESGCM(data_key)
+                with open(output_path, "wb") as out:
+                    out.write(b"\x00\x00\x00\x00")
+                    chunk_index = 0
+                    while True:
+                        chunk = stream.read(self.chunk_size)
+                        if not chunk:
+                            break
+                        chunk_nonce = self._derive_chunk_nonce(base_nonce, chunk_index)
+                        encrypted_chunk = aesgcm.encrypt(chunk_nonce, chunk, None)
+                        out.write(len(encrypted_chunk).to_bytes(self.HEADER_SIZE, "big"))
+                        out.write(encrypted_chunk)
+                        chunk_index += 1
+                    out.seek(0)
+                    out.write(chunk_index.to_bytes(4, "big"))
+
+        return EncryptionMetadata(
+            algorithm="AES256",
+            key_id=self.provider.KEY_ID if hasattr(self.provider, "KEY_ID") else "local",
+            nonce=base_nonce,
+            encrypted_data_key=encrypted_data_key,
+        )
+
+    def decrypt_file(self, input_path: str, output_path: str,
+                     metadata: EncryptionMetadata) -> None:
+        data_key = self.provider.decrypt_data_key(metadata.encrypted_data_key, metadata.key_id)
+        base_nonce = metadata.nonce
+
+        if _HAS_RUST:
+            _rc.decrypt_stream_chunked(input_path, output_path, data_key, base_nonce)
+        else:
+            with open(input_path, "rb") as stream:
+                chunk_count_bytes = stream.read(4)
+                if len(chunk_count_bytes) < 4:
+                    raise EncryptionError("Invalid encrypted stream: missing header")
+                chunk_count = int.from_bytes(chunk_count_bytes, "big")
+                aesgcm = AESGCM(data_key)
+                with open(output_path, "wb") as out:
+                    for chunk_index in range(chunk_count):
+                        size_bytes = stream.read(self.HEADER_SIZE)
+                        if len(size_bytes) < self.HEADER_SIZE:
+                            raise EncryptionError(f"Invalid encrypted stream: truncated at chunk {chunk_index}")
+                        chunk_size = int.from_bytes(size_bytes, "big")
+                        encrypted_chunk = stream.read(chunk_size)
+                        if len(encrypted_chunk) < chunk_size:
+                            raise EncryptionError(f"Invalid encrypted stream: incomplete chunk {chunk_index}")
+                        chunk_nonce = self._derive_chunk_nonce(base_nonce, chunk_index)
+                        try:
+                            decrypted_chunk = aesgcm.decrypt(chunk_nonce, encrypted_chunk, None)
+                            out.write(decrypted_chunk)
+                        except Exception as exc:
+                            raise EncryptionError(f"Failed to decrypt chunk {chunk_index}: {exc}") from exc
+
 
 class EncryptionManager:
     """Manages encryption providers and operations."""

app/iam.py

@@ -164,9 +164,14 @@ class IamService:
         self._clear_failed_attempts(access_key)
         return self._build_principal(access_key, record)
 
+    _MAX_LOCKOUT_KEYS = 10000
+
     def _record_failed_attempt(self, access_key: str) -> None:
         if not access_key:
             return
+        if access_key not in self._failed_attempts and len(self._failed_attempts) >= self._MAX_LOCKOUT_KEYS:
+            oldest_key = min(self._failed_attempts, key=lambda k: self._failed_attempts[k][0] if self._failed_attempts[k] else datetime.min.replace(tzinfo=timezone.utc))
+            del self._failed_attempts[oldest_key]
         attempts = self._failed_attempts.setdefault(access_key, deque())
         self._prune_attempts(attempts)
         attempts.append(datetime.now(timezone.utc))

app/notifications.py

@@ -15,29 +15,23 @@ from typing import Any, Dict, List, Optional
 from urllib.parse import urlparse
 
 import requests
+from urllib3.util.connection import create_connection as _urllib3_create_connection
 
 
-def _is_safe_url(url: str, allow_internal: bool = False) -> bool:
-    """Check if a URL is safe to make requests to (not internal/private).
-
-    Args:
-        url: The URL to check.
-        allow_internal: If True, allows internal/private IP addresses.
-                       Use for self-hosted deployments on internal networks.
-    """
+def _resolve_and_check_url(url: str, allow_internal: bool = False) -> Optional[str]:
     try:
         parsed = urlparse(url)
         hostname = parsed.hostname
         if not hostname:
-            return False
+            return None
         cloud_metadata_hosts = {
             "metadata.google.internal",
             "169.254.169.254",
         }
         if hostname.lower() in cloud_metadata_hosts:
-            return False
+            return None
         if allow_internal:
-            return True
+            return hostname
         blocked_hosts = {
             "localhost",
             "127.0.0.1",
@@ -46,17 +40,46 @@ def _is_safe_url(url: str, allow_internal: bool = False) -> bool:
             "[::1]",
         }
         if hostname.lower() in blocked_hosts:
-            return False
+            return None
         try:
             resolved_ip = socket.gethostbyname(hostname)
             ip = ipaddress.ip_address(resolved_ip)
             if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
-                return False
+                return None
+            return resolved_ip
         except (socket.gaierror, ValueError):
-            return False
-        return True
+            return None
     except Exception:
-        return False
+        return None
+
+
+def _is_safe_url(url: str, allow_internal: bool = False) -> bool:
+    return _resolve_and_check_url(url, allow_internal) is not None
+
+
+_dns_pin_lock = threading.Lock()
+
+
+def _pinned_post(url: str, pinned_ip: str, **kwargs: Any) -> requests.Response:
+    parsed = urlparse(url)
+    hostname = parsed.hostname or ""
+    session = requests.Session()
+    original_create = _urllib3_create_connection
+
+    def _create_pinned(address: Any, *args: Any, **kw: Any) -> Any:
+        host, req_port = address
+        if host == hostname:
+            return original_create((pinned_ip, req_port), *args, **kw)
+        return original_create(address, *args, **kw)
+
+    import urllib3.util.connection as _conn_mod
+    with _dns_pin_lock:
+        _conn_mod.create_connection = _create_pinned
+        try:
+            return session.post(url, **kwargs)
+        finally:
+            _conn_mod.create_connection = original_create
+
 
 logger = logging.getLogger(__name__)
 
@@ -344,16 +367,18 @@ class NotificationService:
                 self._queue.task_done()
 
     def _send_notification(self, event: NotificationEvent, destination: WebhookDestination) -> None:
-        if not _is_safe_url(destination.url, allow_internal=self._allow_internal_endpoints):
-            raise RuntimeError(f"Blocked request to cloud metadata service (SSRF protection): {destination.url}")
+        resolved_ip = _resolve_and_check_url(destination.url, allow_internal=self._allow_internal_endpoints)
+        if not resolved_ip:
+            raise RuntimeError(f"Blocked request (SSRF protection): {destination.url}")
         payload = event.to_s3_event()
         headers = {"Content-Type": "application/json", **destination.headers}
 
         last_error = None
         for attempt in range(destination.retry_count):
             try:
-                response = requests.post(
+                response = _pinned_post(
                     destination.url,
+                    resolved_ip,
                     json=payload,
                     headers=headers,
                     timeout=destination.timeout_seconds,

app/s3_api.py

@@ -267,39 +267,6 @@ def _verify_sigv4_header(req: Any, auth_header: str) -> Principal | None:
     if not secret_key:
         raise IamError("SignatureDoesNotMatch")
 
-    method = req.method
-    canonical_uri = _get_canonical_uri(req)
-    
-    query_args = []
-    for key, value in req.args.items(multi=True):
-        query_args.append((key, value))
-    query_args.sort(key=lambda x: (x[0], x[1]))
-    
-    canonical_query_parts = []
-    for k, v in query_args:
-        canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}")
-    canonical_query_string = "&".join(canonical_query_parts)
-
-    signed_headers_list = signed_headers_str.split(";")
-    canonical_headers_parts = []
-    for header in signed_headers_list:
-        header_val = req.headers.get(header)
-        if header_val is None:
-             header_val = ""
-        
-        if header.lower() == 'expect' and header_val == "":
-            header_val = "100-continue"
-        
-        header_val = " ".join(header_val.split())
-        canonical_headers_parts.append(f"{header.lower()}:{header_val}\n")
-    canonical_headers = "".join(canonical_headers_parts)
-
-    payload_hash = req.headers.get("X-Amz-Content-Sha256")
-    if not payload_hash:
-        payload_hash = hashlib.sha256(req.get_data()).hexdigest()
-
-    canonical_request = f"{method}\n{canonical_uri}\n{canonical_query_string}\n{canonical_headers}\n{signed_headers_str}\n{payload_hash}"
-
     amz_date = req.headers.get("X-Amz-Date") or req.headers.get("Date")
     if not amz_date:
         raise IamError("Missing Date header")
@@ -321,23 +288,50 @@ def _verify_sigv4_header(req: Any, auth_header: str) -> Principal | None:
         if 'date' in signed_headers_set:
             required_headers.remove('x-amz-date')
             required_headers.add('date')
-        
+
         if not required_headers.issubset(signed_headers_set):
              raise IamError("Required headers not signed")
 
-    credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
-    signing_key = _get_signature_key(secret_key, date_stamp, region, service)
+    canonical_uri = _get_canonical_uri(req)
+    payload_hash = req.headers.get("X-Amz-Content-Sha256") or "UNSIGNED-PAYLOAD"
+
     if _HAS_RUST:
-        string_to_sign = _rc.build_string_to_sign(amz_date, credential_scope, canonical_request)
-        calculated_signature = _rc.compute_signature(signing_key, string_to_sign)
+        query_params = list(req.args.items(multi=True))
+        header_values = [(h, req.headers.get(h) or "") for h in signed_headers_str.split(";")]
+        if not _rc.verify_sigv4_signature(
+            req.method, canonical_uri, query_params, signed_headers_str,
+            header_values, payload_hash, amz_date, date_stamp, region,
+            service, secret_key, signature,
+        ):
+            raise IamError("SignatureDoesNotMatch")
     else:
+        method = req.method
+        query_args = sorted(req.args.items(multi=True), key=lambda x: (x[0], x[1]))
+        canonical_query_parts = []
+        for k, v in query_args:
+            canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}")
+        canonical_query_string = "&".join(canonical_query_parts)
+
+        signed_headers_list = signed_headers_str.split(";")
+        canonical_headers_parts = []
+        for header in signed_headers_list:
+            header_val = req.headers.get(header)
+            if header_val is None:
+                 header_val = ""
+            if header.lower() == 'expect' and header_val == "":
+                header_val = "100-continue"
+            header_val = " ".join(header_val.split())
+            canonical_headers_parts.append(f"{header.lower()}:{header_val}\n")
+        canonical_headers = "".join(canonical_headers_parts)
+
+        canonical_request = f"{method}\n{canonical_uri}\n{canonical_query_string}\n{canonical_headers}\n{signed_headers_str}\n{payload_hash}"
+
+        credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
+        signing_key = _get_signature_key(secret_key, date_stamp, region, service)
         string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}"
         calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
-
-    if not hmac.compare_digest(calculated_signature, signature):
-        if current_app.config.get("DEBUG_SIGV4"):
-            logger.warning("SigV4 signature mismatch for %s %s", method, req.path)
-        raise IamError("SignatureDoesNotMatch")
+        if not hmac.compare_digest(calculated_signature, signature):
+            raise IamError("SignatureDoesNotMatch")
 
     session_token = req.headers.get("X-Amz-Security-Token")
     if session_token:
@@ -366,14 +360,21 @@ def _verify_sigv4_query(req: Any) -> Principal | None:
         req_time = datetime.strptime(amz_date, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc)
     except ValueError:
         raise IamError("Invalid Date format")
-    
+
     now = datetime.now(timezone.utc)
+    tolerance = timedelta(seconds=current_app.config.get("SIGV4_TIMESTAMP_TOLERANCE_SECONDS", 900))
+    if req_time > now + tolerance:
+        raise IamError("Request date is too far in the future")
     try:
         expires_seconds = int(expires)
         if expires_seconds <= 0:
             raise IamError("Invalid Expires value: must be positive")
     except ValueError:
         raise IamError("Invalid Expires value: must be an integer")
+    min_expiry = current_app.config.get("PRESIGNED_URL_MIN_EXPIRY_SECONDS", 1)
+    max_expiry = current_app.config.get("PRESIGNED_URL_MAX_EXPIRY_SECONDS", 604800)
+    if expires_seconds < min_expiry or expires_seconds > max_expiry:
+        raise IamError(f"Expiration must be between {min_expiry} second(s) and {max_expiry} seconds")
     if now > req_time + timedelta(seconds=expires_seconds):
         raise IamError("Request expired")
 
@@ -381,53 +382,58 @@ def _verify_sigv4_query(req: Any) -> Principal | None:
     if not secret_key:
         raise IamError("Invalid access key")
 
-    method = req.method
     canonical_uri = _get_canonical_uri(req)
-    
-    query_args = []
-    for key, value in req.args.items(multi=True):
-        if key != "X-Amz-Signature":
-            query_args.append((key, value))
-    query_args.sort(key=lambda x: (x[0], x[1]))
-    
-    canonical_query_parts = []
-    for k, v in query_args:
-        canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}")
-    canonical_query_string = "&".join(canonical_query_parts)
-    
-    signed_headers_list = signed_headers_str.split(";")
-    canonical_headers_parts = []
-    for header in signed_headers_list:
-        val = req.headers.get(header, "").strip()
-        if header.lower() == 'expect' and val == "":
-            val = "100-continue"
-        val = " ".join(val.split())
-        canonical_headers_parts.append(f"{header.lower()}:{val}\n")
-    canonical_headers = "".join(canonical_headers_parts)
-    
-    payload_hash = "UNSIGNED-PAYLOAD"
-    
-    canonical_request = "\n".join([
-        method,
-        canonical_uri,
-        canonical_query_string,
-        canonical_headers,
-        signed_headers_str,
-        payload_hash
-    ])
-    
-    credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
-    signing_key = _get_signature_key(secret_key, date_stamp, region, service)
+
     if _HAS_RUST:
-        string_to_sign = _rc.build_string_to_sign(amz_date, credential_scope, canonical_request)
-        calculated_signature = _rc.compute_signature(signing_key, string_to_sign)
+        query_params = [(k, v) for k, v in req.args.items(multi=True) if k != "X-Amz-Signature"]
+        header_values = [(h, req.headers.get(h) or "") for h in signed_headers_str.split(";")]
+        if not _rc.verify_sigv4_signature(
+            req.method, canonical_uri, query_params, signed_headers_str,
+            header_values, "UNSIGNED-PAYLOAD", amz_date, date_stamp, region,
+            service, secret_key, signature,
+        ):
+            raise IamError("SignatureDoesNotMatch")
     else:
+        method = req.method
+        query_args = []
+        for key, value in req.args.items(multi=True):
+            if key != "X-Amz-Signature":
+                query_args.append((key, value))
+        query_args.sort(key=lambda x: (x[0], x[1]))
+
+        canonical_query_parts = []
+        for k, v in query_args:
+            canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}")
+        canonical_query_string = "&".join(canonical_query_parts)
+
+        signed_headers_list = signed_headers_str.split(";")
+        canonical_headers_parts = []
+        for header in signed_headers_list:
+            val = req.headers.get(header, "").strip()
+            if header.lower() == 'expect' and val == "":
+                val = "100-continue"
+            val = " ".join(val.split())
+            canonical_headers_parts.append(f"{header.lower()}:{val}\n")
+        canonical_headers = "".join(canonical_headers_parts)
+
+        payload_hash = "UNSIGNED-PAYLOAD"
+
+        canonical_request = "\n".join([
+            method,
+            canonical_uri,
+            canonical_query_string,
+            canonical_headers,
+            signed_headers_str,
+            payload_hash
+        ])
+
+        credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
+        signing_key = _get_signature_key(secret_key, date_stamp, region, service)
         hashed_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
         string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashed_request}"
         calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
-
-    if not hmac.compare_digest(calculated_signature, signature):
-        raise IamError("SignatureDoesNotMatch")
+        if not hmac.compare_digest(calculated_signature, signature):
+            raise IamError("SignatureDoesNotMatch")
 
     session_token = req.args.get("X-Amz-Security-Token")
     if session_token:
@@ -586,7 +592,11 @@ def _validate_presigned_request(action: str, bucket_name: str, object_key: str)
         request_time = datetime.strptime(amz_date, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc)
     except ValueError as exc:
         raise IamError("Invalid X-Amz-Date") from exc
-    if datetime.now(timezone.utc) > request_time + timedelta(seconds=expiry):
+    now = datetime.now(timezone.utc)
+    tolerance = timedelta(seconds=current_app.config.get("SIGV4_TIMESTAMP_TOLERANCE_SECONDS", 900))
+    if request_time > now + tolerance:
+        raise IamError("Request date is too far in the future")
+    if now > request_time + timedelta(seconds=expiry):
         raise IamError("Presigned URL expired")
 
     signed_headers_list = [header.strip().lower() for header in signed_headers.split(";") if header]
@@ -662,7 +672,7 @@ def _extract_request_metadata() -> Dict[str, str]:
     for header, value in request.headers.items():
         if header.lower().startswith("x-amz-meta-"):
             key = header[11:]
-            if key:
+            if key and not (key.startswith("__") and key.endswith("__")):
                 metadata[key] = value
     return metadata
 
@@ -986,7 +996,7 @@ def _render_encryption_document(config: dict[str, Any]) -> Element:
     return root
 
 
-def _stream_file(path, chunk_size: int = 64 * 1024):
+def _stream_file(path, chunk_size: int = 256 * 1024):
     with path.open("rb") as handle:
         while True:
             chunk = handle.read(chunk_size)
@@ -1019,6 +1029,8 @@ def _apply_object_headers(
     response.headers["ETag"] = f'"{etag}"'
     response.headers["Accept-Ranges"] = "bytes"
     for key, value in (metadata or {}).items():
+        if key.startswith("__") and key.endswith("__"):
+            continue
         safe_value = _sanitize_header_value(str(value))
         response.headers[f"X-Amz-Meta-{key}"] = safe_value
 
@@ -1039,6 +1051,7 @@ def _maybe_handle_bucket_subresource(bucket_name: str) -> Response | None:
         "logging": _bucket_logging_handler,
         "uploads": _bucket_uploads_handler,
         "policy": _bucket_policy_handler,
+        "policyStatus": _bucket_policy_status_handler,
         "replication": _bucket_replication_handler,
         "website": _bucket_website_handler,
     }
@@ -1321,8 +1334,8 @@ def _bucket_cors_handler(bucket_name: str) -> Response:
 
 
 def _bucket_encryption_handler(bucket_name: str) -> Response:
-    if request.method not in {"GET", "PUT"}:
-        return _method_not_allowed(["GET", "PUT"])
+    if request.method not in {"GET", "PUT", "DELETE"}:
+        return _method_not_allowed(["GET", "PUT", "DELETE"])
     principal, error = _require_principal()
     if error:
         return error
@@ -1343,6 +1356,13 @@ def _bucket_encryption_handler(bucket_name: str) -> Response:
                 404,
             )
         return _xml_response(_render_encryption_document(config))
+    if request.method == "DELETE":
+        try:
+            storage.set_bucket_encryption(bucket_name, None)
+        except StorageError as exc:
+            return _error_response("NoSuchBucket", str(exc), 404)
+        current_app.logger.info("Bucket encryption deleted", extra={"bucket": bucket_name})
+        return Response(status=204)
     ct_error = _require_xml_content_type()
     if ct_error:
         return ct_error
@@ -1439,6 +1459,99 @@ def _bucket_acl_handler(bucket_name: str) -> Response:
     return _xml_response(root)
 
 
+def _object_acl_handler(bucket_name: str, object_key: str) -> Response:
+    from .acl import create_canned_acl, GRANTEE_ALL_USERS, GRANTEE_AUTHENTICATED_USERS
+
+    if request.method not in {"GET", "PUT"}:
+        return _method_not_allowed(["GET", "PUT"])
+    storage = _storage()
+    try:
+        path = storage.get_object_path(bucket_name, object_key)
+    except (StorageError, FileNotFoundError):
+        return _error_response("NoSuchKey", "Object not found", 404)
+
+    if request.method == "PUT":
+        principal, error = _object_principal("write", bucket_name, object_key)
+        if error:
+            return error
+        owner_id = principal.access_key if principal else "anonymous"
+        canned_acl = request.headers.get("x-amz-acl", "private")
+        acl = create_canned_acl(canned_acl, owner_id)
+        acl_service = _acl()
+        metadata = storage.get_object_metadata(bucket_name, object_key)
+        metadata.update(acl_service.create_object_acl_metadata(acl))
+        safe_key = storage._sanitize_object_key(object_key, storage._object_key_max_length_bytes)
+        storage._write_metadata(bucket_name, safe_key, metadata)
+        current_app.logger.info("Object ACL set", extra={"bucket": bucket_name, "key": object_key, "acl": canned_acl})
+        return Response(status=200)
+
+    principal, error = _object_principal("read", bucket_name, object_key)
+    if error:
+        return error
+    owner_id = principal.access_key if principal else "anonymous"
+    acl_service = _acl()
+    metadata = storage.get_object_metadata(bucket_name, object_key)
+    acl = acl_service.get_object_acl(bucket_name, object_key, metadata)
+    if not acl:
+        acl = create_canned_acl("private", owner_id)
+
+    root = Element("AccessControlPolicy")
+    owner_el = SubElement(root, "Owner")
+    SubElement(owner_el, "ID").text = acl.owner
+    SubElement(owner_el, "DisplayName").text = acl.owner
+    acl_el = SubElement(root, "AccessControlList")
+    for grant in acl.grants:
+        grant_el = SubElement(acl_el, "Grant")
+        grantee = SubElement(grant_el, "Grantee")
+        if grant.grantee == GRANTEE_ALL_USERS:
+            grantee.set("{http://www.w3.org/2001/XMLSchema-instance}type", "Group")
+            SubElement(grantee, "URI").text = "http://acs.amazonaws.com/groups/global/AllUsers"
+        elif grant.grantee == GRANTEE_AUTHENTICATED_USERS:
+            grantee.set("{http://www.w3.org/2001/XMLSchema-instance}type", "Group")
+            SubElement(grantee, "URI").text = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
+        else:
+            grantee.set("{http://www.w3.org/2001/XMLSchema-instance}type", "CanonicalUser")
+            SubElement(grantee, "ID").text = grant.grantee
+            SubElement(grantee, "DisplayName").text = grant.grantee
+        SubElement(grant_el, "Permission").text = grant.permission
+    return _xml_response(root)
+
+
+def _object_attributes_handler(bucket_name: str, object_key: str) -> Response:
+    if request.method != "GET":
+        return _method_not_allowed(["GET"])
+    principal, error = _object_principal("read", bucket_name, object_key)
+    if error:
+        return error
+    storage = _storage()
+    try:
+        path = storage.get_object_path(bucket_name, object_key)
+        file_stat = path.stat()
+        metadata = storage.get_object_metadata(bucket_name, object_key)
+    except (StorageError, FileNotFoundError):
+        return _error_response("NoSuchKey", "Object not found", 404)
+
+    requested = request.headers.get("x-amz-object-attributes", "")
+    attrs = {a.strip() for a in requested.split(",") if a.strip()}
+
+    root = Element("GetObjectAttributesResponse")
+    if "ETag" in attrs:
+        etag = metadata.get("__etag__") or storage._compute_etag(path)
+        SubElement(root, "ETag").text = etag
+    if "StorageClass" in attrs:
+        SubElement(root, "StorageClass").text = "STANDARD"
+    if "ObjectSize" in attrs:
+        SubElement(root, "ObjectSize").text = str(file_stat.st_size)
+    if "Checksum" in attrs:
+        SubElement(root, "Checksum")
+    if "ObjectParts" in attrs:
+        SubElement(root, "ObjectParts")
+
+    response = _xml_response(root)
+    response.headers["Last-Modified"] = http_date(file_stat.st_mtime)
+    return response
+
+
 def _bucket_list_versions_handler(bucket_name: str) -> Response:
     """Handle ListObjectVersions (GET /<bucket>?versions)."""
     if request.method != "GET":
@@ -2346,7 +2459,7 @@ def _post_object(bucket_name: str) -> Response:
     for field_name, value in request.form.items():
         if field_name.lower().startswith("x-amz-meta-"):
             key = field_name[11:]
-            if key:
+            if key and not (key.startswith("__") and key.endswith("__")):
                 metadata[key] = value
     try:
         meta = storage.put_object(bucket_name, object_key, file.stream, metadata=metadata or None)
@@ -2360,6 +2473,10 @@ def _post_object(bucket_name: str) -> Response:
     if success_action_redirect:
         allowed_hosts = current_app.config.get("ALLOWED_REDIRECT_HOSTS", [])
         if not allowed_hosts:
+            current_app.logger.warning(
+                "ALLOWED_REDIRECT_HOSTS not configured, falling back to request Host header. "
+                "Set ALLOWED_REDIRECT_HOSTS for production deployments."
+            )
             allowed_hosts = [request.host]
         parsed = urlparse(success_action_redirect)
         if parsed.scheme not in ("http", "https"):
@@ -2546,54 +2663,43 @@ def bucket_handler(bucket_name: str) -> Response:
     else:
         effective_start = marker
     
-    fetch_keys = max_keys * 10 if delimiter else max_keys
     try:
-        list_result = storage.list_objects(
-            bucket_name,
-            max_keys=fetch_keys,
-            continuation_token=effective_start or None,
-            prefix=prefix or None,
-        )
-        objects = list_result.objects
+        if delimiter:
+            shallow_result = storage.list_objects_shallow(
+                bucket_name,
+                prefix=prefix,
+                delimiter=delimiter,
+                max_keys=max_keys,
+                continuation_token=effective_start or None,
+            )
+            objects = shallow_result.objects
+            common_prefixes = shallow_result.common_prefixes
+            is_truncated = shallow_result.is_truncated
+
+            next_marker = shallow_result.next_continuation_token or ""
+            next_continuation_token = ""
+            if is_truncated and next_marker and list_type == "2":
+                next_continuation_token = base64.urlsafe_b64encode(next_marker.encode()).decode("utf-8")
+        else:
+            list_result = storage.list_objects(
+                bucket_name,
+                max_keys=max_keys,
+                continuation_token=effective_start or None,
+                prefix=prefix or None,
+            )
+            objects = list_result.objects
+            common_prefixes = []
+            is_truncated = list_result.is_truncated
+
+            next_marker = ""
+            next_continuation_token = ""
+            if is_truncated:
+                if objects:
+                    next_marker = objects[-1].key
+                if list_type == "2" and next_marker:
+                    next_continuation_token = base64.urlsafe_b64encode(next_marker.encode()).decode("utf-8")
     except StorageError as exc:
         return _error_response("NoSuchBucket", str(exc), 404)
-    
-    common_prefixes: list[str] = []
-    filtered_objects: list = []
-    if delimiter:
-        seen_prefixes: set[str] = set()
-        for obj in objects:
-            key_after_prefix = obj.key[len(prefix):] if prefix else obj.key
-            if delimiter in key_after_prefix:
-                common_prefix = prefix + key_after_prefix.split(delimiter)[0] + delimiter
-                if common_prefix not in seen_prefixes:
-                    seen_prefixes.add(common_prefix)
-                    common_prefixes.append(common_prefix)
-            else:
-                filtered_objects.append(obj)
-        objects = filtered_objects
-        common_prefixes = sorted(common_prefixes)
-    
-    total_items = len(objects) + len(common_prefixes)
-    is_truncated = total_items > max_keys or list_result.is_truncated
-    
-    if len(objects) >= max_keys:
-        objects = objects[:max_keys]
-        common_prefixes = []
-    else:
-        remaining = max_keys - len(objects)
-        common_prefixes = common_prefixes[:remaining]
-    
-    next_marker = ""
-    next_continuation_token = ""
-    if is_truncated:
-        if objects:
-            next_marker = objects[-1].key
-        elif common_prefixes:
-            next_marker = common_prefixes[-1].rstrip(delimiter) if delimiter else common_prefixes[-1]
-        
-        if list_type == "2" and next_marker:
-            next_continuation_token = base64.urlsafe_b64encode(next_marker.encode()).decode("utf-8")
 
     if list_type == "2":
         root = Element("ListBucketResult")
@@ -2669,6 +2775,12 @@ def object_handler(bucket_name: str, object_key: str):
     if "legal-hold" in request.args:
         return _object_legal_hold_handler(bucket_name, object_key)
 
+    if "acl" in request.args:
+        return _object_acl_handler(bucket_name, object_key)
+
+    if "attributes" in request.args:
+        return _object_attributes_handler(bucket_name, object_key)
+
     if request.method == "POST":
         if "uploads" in request.args:
             return _initiate_multipart_upload(bucket_name, object_key)
@@ -2816,7 +2928,7 @@ def object_handler(bucket_name: str, object_key: str):
                             f.seek(start_pos)
                             remaining = length_to_read
                             while remaining > 0:
-                                chunk_size = min(65536, remaining)
+                                chunk_size = min(262144, remaining)
                                 chunk = f.read(chunk_size)
                                 if not chunk:
                                     break
@@ -2993,6 +3105,32 @@ def _bucket_policy_handler(bucket_name: str) -> Response:
     return Response(status=204)
 
 
+def _bucket_policy_status_handler(bucket_name: str) -> Response:
+    if request.method != "GET":
+        return _method_not_allowed(["GET"])
+    principal, error = _require_principal()
+    if error:
+        return error
+    try:
+        _authorize_action(principal, bucket_name, "policy")
+    except IamError as exc:
+        return _error_response("AccessDenied", str(exc), 403)
+    storage = _storage()
+    if not storage.bucket_exists(bucket_name):
+        return _error_response("NoSuchBucket", "Bucket does not exist", 404)
+    store = _bucket_policies()
+    policy = store.get_policy(bucket_name)
+    is_public = False
+    if policy:
+        for statement in policy.get("Statement", []):
+            if statement.get("Effect") == "Allow" and statement.get("Principal") == "*":
+                is_public = True
+                break
+    root = Element("PolicyStatus")
+    SubElement(root, "IsPublic").text = "TRUE" if is_public else "FALSE"
+    return _xml_response(root)
+
+
 def _bucket_replication_handler(bucket_name: str) -> Response:
     if request.method not in {"GET", "PUT", "DELETE"}:
         return _method_not_allowed(["GET", "PUT", "DELETE"])
@@ -3206,7 +3344,7 @@ def head_object(bucket_name: str, object_key: str) -> Response:
         path = _storage().get_object_path(bucket_name, object_key)
         metadata = _storage().get_object_metadata(bucket_name, object_key)
         stat = path.stat()
-        etag = _storage()._compute_etag(path)
+        etag = metadata.get("__etag__") or _storage()._compute_etag(path)
         
         response = Response(status=200)
         _apply_object_headers(response, file_stat=stat, metadata=metadata, etag=etag)
@@ -3299,8 +3437,8 @@ def _copy_object(dest_bucket: str, dest_key: str, copy_source: str) -> Response:
         if validation_error:
             return _error_response("InvalidArgument", validation_error, 400)
     else:
-        metadata = source_metadata
-    
+        metadata = {k: v for k, v in source_metadata.items() if not (k.startswith("__") and k.endswith("__"))}
+
     try:
         with source_path.open("rb") as stream:
             meta = storage.put_object(

app/s3_client.py

@@ -245,6 +245,7 @@ def stream_objects_ndjson(
     url_templates: dict[str, str],
     display_tz: str = "UTC",
     versioning_enabled: bool = False,
+    delimiter: Optional[str] = None,
 ) -> Generator[str, None, None]:
     meta_line = json.dumps({
         "type": "meta",
@@ -258,11 +259,20 @@ def stream_objects_ndjson(
     kwargs: dict[str, Any] = {"Bucket": bucket_name, "MaxKeys": 1000}
     if prefix:
         kwargs["Prefix"] = prefix
+    if delimiter:
+        kwargs["Delimiter"] = delimiter
 
+    running_count = 0
     try:
         paginator = client.get_paginator("list_objects_v2")
         for page in paginator.paginate(**kwargs):
-            for obj in page.get("Contents", []):
+            for cp in page.get("CommonPrefixes", []):
+                yield json.dumps({
+                    "type": "folder",
+                    "prefix": cp["Prefix"],
+                }) + "\n"
+            page_contents = page.get("Contents", [])
+            for obj in page_contents:
                 last_mod = obj["LastModified"]
                 yield json.dumps({
                     "type": "object",
@@ -273,6 +283,8 @@ def stream_objects_ndjson(
                     "last_modified_iso": format_datetime_iso(last_mod, display_tz),
                     "etag": obj.get("ETag", "").strip('"'),
                 }) + "\n"
+            running_count += len(page_contents)
+            yield json.dumps({"type": "count", "total_count": running_count}) + "\n"
     except ClientError as exc:
         error_msg = exc.response.get("Error", {}).get("Message", "S3 operation failed")
         yield json.dumps({"type": "error", "error": error_msg}) + "\n"

app/storage.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import copy
 import hashlib
 import json
 import os
@@ -15,7 +16,7 @@ from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager
 from dataclasses import dataclass
 from datetime import datetime, timezone
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from typing import Any, BinaryIO, Dict, Generator, List, Optional
 
 try:
@@ -153,6 +154,15 @@ class ListObjectsResult:
     total_count: Optional[int] = None
 
 
+@dataclass
+class ShallowListResult:
+    """Result for delimiter-aware directory-level listing."""
+    objects: List[ObjectMeta]
+    common_prefixes: List[str]
+    is_truncated: bool
+    next_continuation_token: Optional[str]
+
+
 def _utcnow() -> datetime:
     return datetime.now(timezone.utc)
 
@@ -196,6 +206,8 @@ class ObjectStorage:
         self._object_key_max_length_bytes = object_key_max_length_bytes
         self._sorted_key_cache: Dict[str, tuple[list[str], int]] = {}
         self._meta_index_locks: Dict[str, threading.Lock] = {}
+        self._meta_read_cache: OrderedDict[tuple, Optional[Dict[str, Any]]] = OrderedDict()
+        self._meta_read_cache_max = 2048
         self._cleanup_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="ParentCleanup")
 
     def _get_bucket_lock(self, bucket_id: str) -> threading.Lock:
@@ -276,25 +288,47 @@ class ObjectStorage:
         version_count = 0
         version_bytes = 0
 
-        try:
-            for path in bucket_path.rglob("*"):
-                if path.is_file():
-                    rel = path.relative_to(bucket_path)
-                    if not rel.parts:
-                        continue
-                    top_folder = rel.parts[0]
-                    if top_folder not in self.INTERNAL_FOLDERS:
-                        stat = path.stat()
-                        object_count += 1
-                        total_bytes += stat.st_size
+        internal = self.INTERNAL_FOLDERS
+        bucket_str = str(bucket_path)
 
-            versions_root = self._bucket_versions_root(bucket_name)
-            if versions_root.exists():
-                for path in versions_root.rglob("*.bin"):
-                    if path.is_file():
-                        stat = path.stat()
-                        version_count += 1
-                        version_bytes += stat.st_size
+        try:
+            if _HAS_RUST:
+                versions_root = str(self._bucket_versions_root(bucket_name))
+                object_count, total_bytes, version_count, version_bytes = _rc.bucket_stats_scan(
+                    bucket_str, versions_root
+                )
+            else:
+                stack = [bucket_str]
+                while stack:
+                    current = stack.pop()
+                    try:
+                        with os.scandir(current) as it:
+                            for entry in it:
+                                if current == bucket_str and entry.name in internal:
+                                    continue
+                                if entry.is_dir(follow_symlinks=False):
+                                    stack.append(entry.path)
+                                elif entry.is_file(follow_symlinks=False):
+                                    object_count += 1
+                                    total_bytes += entry.stat(follow_symlinks=False).st_size
+                    except PermissionError:
+                        continue
+
+                versions_root = self._bucket_versions_root(bucket_name)
+                if versions_root.exists():
+                    v_stack = [str(versions_root)]
+                    while v_stack:
+                        v_current = v_stack.pop()
+                        try:
+                            with os.scandir(v_current) as it:
+                                for entry in it:
+                                    if entry.is_dir(follow_symlinks=False):
+                                        v_stack.append(entry.path)
+                                    elif entry.is_file(follow_symlinks=False) and entry.name.endswith(".bin"):
+                                        version_count += 1
+                                        version_bytes += entry.stat(follow_symlinks=False).st_size
+                        except PermissionError:
+                            continue
         except OSError:
             if cached_stats is not None:
                 return cached_stats
@@ -374,9 +408,18 @@ class ObjectStorage:
             raise StorageError("Bucket contains archived object versions")
         if has_multipart:
             raise StorageError("Bucket has active multipart uploads")
+        bucket_id = bucket_path.name
         self._remove_tree(bucket_path)
-        self._remove_tree(self._system_bucket_root(bucket_path.name))
-        self._remove_tree(self._multipart_bucket_root(bucket_path.name))
+        self._remove_tree(self._system_bucket_root(bucket_id))
+        self._remove_tree(self._multipart_bucket_root(bucket_id))
+        self._bucket_config_cache.pop(bucket_id, None)
+        with self._cache_lock:
+            self._object_cache.pop(bucket_id, None)
+            self._cache_version.pop(bucket_id, None)
+            self._sorted_key_cache.pop(bucket_id, None)
+            stale = [k for k in self._meta_read_cache if k[0] == bucket_id]
+            for k in stale:
+                del self._meta_read_cache[k]
 
     def list_objects(
         self,
@@ -459,6 +502,317 @@ class ObjectStorage:
         result = self.list_objects(bucket_name, max_keys=100000)
         return result.objects
 
+    def list_objects_shallow(
+        self,
+        bucket_name: str,
+        *,
+        prefix: str = "",
+        delimiter: str = "/",
+        max_keys: int = 1000,
+        continuation_token: Optional[str] = None,
+    ) -> ShallowListResult:
+        import bisect
+
+        bucket_path = self._bucket_path(bucket_name)
+        if not bucket_path.exists():
+            raise BucketNotFoundError("Bucket does not exist")
+        bucket_id = bucket_path.name
+
+        if delimiter != "/" or (prefix and not prefix.endswith(delimiter)):
+            return self._shallow_via_full_scan(
+                bucket_name, prefix=prefix, delimiter=delimiter,
+                max_keys=max_keys, continuation_token=continuation_token,
+            )
+
+        target_dir = bucket_path
+        if prefix:
+            safe_prefix_path = Path(prefix.rstrip("/"))
+            if ".." in safe_prefix_path.parts:
+                return ShallowListResult(
+                    objects=[], common_prefixes=[],
+                    is_truncated=False, next_continuation_token=None,
+                )
+            target_dir = bucket_path / safe_prefix_path
+            try:
+                resolved = target_dir.resolve()
+                bucket_resolved = bucket_path.resolve()
+                if not str(resolved).startswith(str(bucket_resolved) + os.sep) and resolved != bucket_resolved:
+                    return ShallowListResult(
+                        objects=[], common_prefixes=[],
+                        is_truncated=False, next_continuation_token=None,
+                    )
+            except (OSError, ValueError):
+                return ShallowListResult(
+                    objects=[], common_prefixes=[],
+                    is_truncated=False, next_continuation_token=None,
+                )
+
+        if not target_dir.exists() or not target_dir.is_dir():
+            return ShallowListResult(
+                objects=[], common_prefixes=[],
+                is_truncated=False, next_continuation_token=None,
+            )
+
+        etag_index_path = self._system_bucket_root(bucket_id) / "etag_index.json"
+        meta_cache: Dict[str, str] = {}
+        if etag_index_path.exists():
+            try:
+                with open(etag_index_path, 'r', encoding='utf-8') as f:
+                    meta_cache = json.load(f)
+            except (OSError, json.JSONDecodeError):
+                pass
+
+        entries_files: list[tuple[str, int, float, Optional[str]]] = []
+        entries_dirs: list[str] = []
+
+        if _HAS_RUST:
+            try:
+                raw = _rc.shallow_scan(str(target_dir), prefix, json.dumps(meta_cache))
+                entries_files = []
+                for key, size, mtime, etag in raw["files"]:
+                    if etag is None:
+                        safe_key = PurePosixPath(key)
+                        meta = self._read_metadata(bucket_id, Path(safe_key))
+                        etag = meta.get("__etag__") if meta else None
+                    entries_files.append((key, size, mtime, etag))
+                entries_dirs = raw["dirs"]
+                all_items = raw["merged_keys"]
+            except OSError:
+                return ShallowListResult(
+                    objects=[], common_prefixes=[],
+                    is_truncated=False, next_continuation_token=None,
+                )
+        else:
+            try:
+                with os.scandir(str(target_dir)) as it:
+                    for entry in it:
+                        name = entry.name
+                        if name in self.INTERNAL_FOLDERS:
+                            continue
+                        if entry.is_dir(follow_symlinks=False):
+                            cp = prefix + name + delimiter
+                            entries_dirs.append(cp)
+                        elif entry.is_file(follow_symlinks=False):
+                            key = prefix + name
+                            try:
+                                st = entry.stat()
+                                etag = meta_cache.get(key)
+                                if etag is None:
+                                    safe_key = PurePosixPath(key)
+                                    meta = self._read_metadata(bucket_id, Path(safe_key))
+                                    etag = meta.get("__etag__") if meta else None
+                                entries_files.append((key, st.st_size, st.st_mtime, etag))
+                            except OSError:
+                                pass
+            except OSError:
+                return ShallowListResult(
+                    objects=[], common_prefixes=[],
+                    is_truncated=False, next_continuation_token=None,
+                )
+
+            entries_dirs.sort()
+            entries_files.sort(key=lambda x: x[0])
+
+            all_items: list[tuple[str, bool]] = []
+            fi, di = 0, 0
+            while fi < len(entries_files) and di < len(entries_dirs):
+                if entries_files[fi][0] <= entries_dirs[di]:
+                    all_items.append((entries_files[fi][0], False))
+                    fi += 1
+                else:
+                    all_items.append((entries_dirs[di], True))
+                    di += 1
+            while fi < len(entries_files):
+                all_items.append((entries_files[fi][0], False))
+                fi += 1
+            while di < len(entries_dirs):
+                all_items.append((entries_dirs[di], True))
+                di += 1
+
+        files_map = {e[0]: e for e in entries_files}
+
+        start_index = 0
+        if continuation_token:
+            all_keys = [item[0] for item in all_items]
+            start_index = bisect.bisect_right(all_keys, continuation_token)
+
+        selected = all_items[start_index:start_index + max_keys]
+        is_truncated = (start_index + max_keys) < len(all_items)
+
+        result_objects: list[ObjectMeta] = []
+        result_prefixes: list[str] = []
+        for item_key, is_dir in selected:
+            if is_dir:
+                result_prefixes.append(item_key)
+            else:
+                fdata = files_map[item_key]
+                result_objects.append(ObjectMeta(
+                    key=fdata[0],
+                    size=fdata[1],
+                    last_modified=datetime.fromtimestamp(fdata[2], timezone.utc),
+                    etag=fdata[3],
+                    metadata=None,
+                ))
+
+        next_token = None
+        if is_truncated and selected:
+            next_token = selected[-1][0]
+
+        return ShallowListResult(
+            objects=result_objects,
+            common_prefixes=result_prefixes,
+            is_truncated=is_truncated,
+            next_continuation_token=next_token,
+        )
+
+    def _shallow_via_full_scan(
+        self,
+        bucket_name: str,
+        *,
+        prefix: str = "",
+        delimiter: str = "/",
+        max_keys: int = 1000,
+        continuation_token: Optional[str] = None,
+    ) -> ShallowListResult:
+        list_result = self.list_objects(
+            bucket_name,
+            max_keys=max_keys * 10,
+            continuation_token=continuation_token,
+            prefix=prefix or None,
+        )
+
+        common_prefixes: list[str] = []
+        filtered_objects: list[ObjectMeta] = []
+        seen_prefixes: set[str] = set()
+
+        for obj in list_result.objects:
+            key_after_prefix = obj.key[len(prefix):] if prefix else obj.key
+            if delimiter in key_after_prefix:
+                cp = prefix + key_after_prefix.split(delimiter)[0] + delimiter
+                if cp not in seen_prefixes:
+                    seen_prefixes.add(cp)
+                    common_prefixes.append(cp)
+            else:
+                filtered_objects.append(obj)
+
+        common_prefixes.sort()
+        total_items = len(filtered_objects) + len(common_prefixes)
+        is_truncated = total_items > max_keys or list_result.is_truncated
+
+        if len(filtered_objects) >= max_keys:
+            filtered_objects = filtered_objects[:max_keys]
+            common_prefixes = []
+        else:
+            remaining = max_keys - len(filtered_objects)
+            common_prefixes = common_prefixes[:remaining]
+
+        next_token = None
+        if is_truncated:
+            if filtered_objects:
+                next_token = filtered_objects[-1].key
+            elif common_prefixes:
+                next_token = common_prefixes[-1].rstrip(delimiter) if delimiter else common_prefixes[-1]
+
+        return ShallowListResult(
+            objects=filtered_objects,
+            common_prefixes=common_prefixes,
+            is_truncated=is_truncated,
+            next_continuation_token=next_token,
+        )
+
+    def search_objects(
+        self,
+        bucket_name: str,
+        query: str,
+        *,
+        prefix: str = "",
+        limit: int = 500,
+    ) -> Dict[str, Any]:
+        bucket_path = self._bucket_path(bucket_name)
+        if not bucket_path.is_dir():
+            raise BucketNotFoundError("Bucket does not exist")
+
+        if prefix:
+            search_root = bucket_path / prefix.replace("/", os.sep)
+            if not search_root.is_dir():
+                return {"results": [], "truncated": False}
+            resolved = search_root.resolve()
+            if not str(resolved).startswith(str(bucket_path.resolve())):
+                return {"results": [], "truncated": False}
+        else:
+            search_root = bucket_path
+
+        if _HAS_RUST:
+            raw = _rc.search_objects_scan(
+                str(bucket_path), str(search_root), query, limit
+            )
+            results = [
+                {
+                    "key": k,
+                    "size": s,
+                    "last_modified": datetime.fromtimestamp(
+                        m, tz=timezone.utc
+                    ).strftime("%Y-%m-%dT%H:%M:%S.000Z"),
+                }
+                for k, s, m in raw["results"]
+            ]
+            return {"results": results, "truncated": raw["truncated"]}
+
+        query_lower = query.lower()
+        results: list[Dict[str, Any]] = []
+        internal = self.INTERNAL_FOLDERS
+        bucket_str = str(bucket_path)
+        bucket_len = len(bucket_str) + 1
+        meta_root = self._bucket_meta_root(bucket_name)
+        scan_limit = limit * 4
+
+        matched = 0
+        scanned = 0
+        search_str = str(search_root)
+        stack = [search_str]
+        while stack:
+            current = stack.pop()
+            try:
+                with os.scandir(current) as it:
+                    for entry in it:
+                        if current == bucket_str and entry.name in internal:
+                            continue
+                        if entry.is_dir(follow_symlinks=False):
+                            stack.append(entry.path)
+                        elif entry.is_file(follow_symlinks=False):
+                            scanned += 1
+                            key = entry.path[bucket_len:].replace(os.sep, "/")
+                            if query_lower in key.lower():
+                                st = entry.stat(follow_symlinks=False)
+                                meta_path = meta_root / (key + ".meta.json")
+                                last_modified = ""
+                                try:
+                                    if meta_path.exists():
+                                        md = json.loads(meta_path.read_text(encoding="utf-8"))
+                                        last_modified = md.get("last_modified", "")
+                                except (OSError, json.JSONDecodeError):
+                                    pass
+                                if not last_modified:
+                                    last_modified = datetime.fromtimestamp(
+                                        st.st_mtime, tz=timezone.utc
+                                    ).strftime("%Y-%m-%dT%H:%M:%S.000Z")
+                                results.append({
+                                    "key": key,
+                                    "size": st.st_size,
+                                    "last_modified": last_modified,
+                                })
+                                matched += 1
+                            if matched >= scan_limit:
+                                break
+            except PermissionError:
+                continue
+            if matched >= scan_limit:
+                break
+
+        results.sort(key=lambda r: r["key"])
+        truncated = len(results) > limit
+        return {"results": results[:limit], "truncated": truncated}
+
     def put_object(
         self,
         bucket_name: str,
@@ -487,40 +841,78 @@ class ObjectStorage:
 
         tmp_dir = self._system_root_path() / self.SYSTEM_TMP_DIR
         tmp_dir.mkdir(parents=True, exist_ok=True)
-        tmp_path = tmp_dir / f"{uuid.uuid4().hex}.tmp"
-        
-        try:
-            checksum = hashlib.md5()
-            with tmp_path.open("wb") as target:
-                shutil.copyfileobj(_HashingReader(stream, checksum), target)
-            
-            new_size = tmp_path.stat().st_size
-            size_delta = new_size - existing_size
-            object_delta = 0 if is_overwrite else 1
 
-            if enforce_quota:
-                quota_check = self.check_quota(
-                    bucket_name,
-                    additional_bytes=max(0, size_delta),
-                    additional_objects=object_delta,
-                )
-                if not quota_check["allowed"]:
-                    raise QuotaExceededError(
-                        quota_check["message"] or "Quota exceeded",
-                        quota_check["quota"],
-                        quota_check["usage"],
-                    )
-
-            shutil.move(str(tmp_path), str(destination))
-            
-        finally:
+        if _HAS_RUST:
+            tmp_path = None
             try:
-                tmp_path.unlink(missing_ok=True)
-            except OSError:
-                pass
+                tmp_path_str, etag, new_size = _rc.stream_to_file_with_md5(
+                    stream, str(tmp_dir)
+                )
+                tmp_path = Path(tmp_path_str)
+
+                size_delta = new_size - existing_size
+                object_delta = 0 if is_overwrite else 1
+
+                if enforce_quota:
+                    quota_check = self.check_quota(
+                        bucket_name,
+                        additional_bytes=max(0, size_delta),
+                        additional_objects=object_delta,
+                    )
+                    if not quota_check["allowed"]:
+                        raise QuotaExceededError(
+                            quota_check["message"] or "Quota exceeded",
+                            quota_check["quota"],
+                            quota_check["usage"],
+                        )
+
+                shutil.move(str(tmp_path), str(destination))
+            finally:
+                if tmp_path:
+                    try:
+                        tmp_path.unlink(missing_ok=True)
+                    except OSError:
+                        pass
+        else:
+            tmp_path = tmp_dir / f"{uuid.uuid4().hex}.tmp"
+            try:
+                with tmp_path.open("wb") as target:
+                    shutil.copyfileobj(stream, target)
+
+                new_size = tmp_path.stat().st_size
+                size_delta = new_size - existing_size
+                object_delta = 0 if is_overwrite else 1
+
+                if enforce_quota:
+                    quota_check = self.check_quota(
+                        bucket_name,
+                        additional_bytes=max(0, size_delta),
+                        additional_objects=object_delta,
+                    )
+                    if not quota_check["allowed"]:
+                        raise QuotaExceededError(
+                            quota_check["message"] or "Quota exceeded",
+                            quota_check["quota"],
+                            quota_check["usage"],
+                        )
+
+                checksum = hashlib.md5()
+                with tmp_path.open("rb") as f:
+                    while True:
+                        chunk = f.read(1048576)
+                        if not chunk:
+                            break
+                        checksum.update(chunk)
+                etag = checksum.hexdigest()
+
+                shutil.move(str(tmp_path), str(destination))
+            finally:
+                try:
+                    tmp_path.unlink(missing_ok=True)
+                except OSError:
+                    pass
 
         stat = destination.stat()
-        etag = checksum.hexdigest()
         
         internal_meta = {"__etag__": etag, "__size__": str(stat.st_size)}
         combined_meta = {**internal_meta, **(metadata or {})}
@@ -959,13 +1351,19 @@ class ObjectStorage:
             version_bytes_delta=archived_version_size,
             version_count_delta=1 if archived_version_size > 0 else 0,
         )
-        return ObjectMeta(
+        etag = self._compute_etag(destination)
+        internal_meta = {"__etag__": etag, "__size__": str(stat.st_size)}
+        combined_meta = {**internal_meta, **(metadata or {})}
+        self._write_metadata(bucket_id, safe_key, combined_meta)
+        obj_meta = ObjectMeta(
             key=safe_key.as_posix(),
             size=stat.st_size,
             last_modified=datetime.fromtimestamp(stat.st_mtime, timezone.utc),
-            etag=self._compute_etag(destination),
+            etag=etag,
             metadata=metadata or None,
         )
+        self._update_object_cache_entry(bucket_id, safe_key.as_posix(), obj_meta)
+        return obj_meta
 
     def delete_object_version(self, bucket_name: str, object_key: str, version_id: str) -> None:
         bucket_path = self._bucket_path(bucket_name)
@@ -1105,14 +1503,24 @@ class ObjectStorage:
         if not upload_root.exists():
             raise StorageError("Multipart upload not found")
 
-        checksum = hashlib.md5()
         part_filename = f"part-{part_number:05d}.part"
         part_path = upload_root / part_filename
         temp_path = upload_root / f".{part_filename}.tmp"
 
         try:
             with temp_path.open("wb") as target:
-                shutil.copyfileobj(_HashingReader(stream, checksum), target)
+                shutil.copyfileobj(stream, target)
+            if _HAS_RUST:
+                part_etag = _rc.md5_file(str(temp_path))
+            else:
+                checksum = hashlib.md5()
+                with temp_path.open("rb") as f:
+                    while True:
+                        chunk = f.read(1048576)
+                        if not chunk:
+                            break
+                        checksum.update(chunk)
+                part_etag = checksum.hexdigest()
             temp_path.replace(part_path)
         except OSError:
             try:
@@ -1122,7 +1530,7 @@ class ObjectStorage:
             raise
 
         record = {
-            "etag": checksum.hexdigest(),
+            "etag": part_etag,
             "size": part_path.stat().st_size,
             "filename": part_filename,
         }
@@ -1322,19 +1730,29 @@ class ObjectStorage:
                 if versioning_enabled and destination.exists():
                     archived_version_size = destination.stat().st_size
                     self._archive_current_version(bucket_id, safe_key, reason="overwrite")
-                checksum = hashlib.md5()
-                with destination.open("wb") as target:
+                if _HAS_RUST:
+                    part_paths = []
                     for _, record in validated:
-                        part_path = upload_root / record["filename"]
-                        if not part_path.exists():
+                        pp = upload_root / record["filename"]
+                        if not pp.exists():
                             raise StorageError(f"Missing part file {record['filename']}")
-                        with part_path.open("rb") as chunk:
-                            while True:
-                                data = chunk.read(1024 * 1024)
-                                if not data:
-                                    break
-                                checksum.update(data)
-                                target.write(data)
+                        part_paths.append(str(pp))
+                    checksum_hex = _rc.assemble_parts_with_md5(part_paths, str(destination))
+                else:
+                    checksum = hashlib.md5()
+                    with destination.open("wb") as target:
+                        for _, record in validated:
+                            part_path = upload_root / record["filename"]
+                            if not part_path.exists():
+                                raise StorageError(f"Missing part file {record['filename']}")
+                            with part_path.open("rb") as chunk:
+                                while True:
+                                    data = chunk.read(1024 * 1024)
+                                    if not data:
+                                        break
+                                    checksum.update(data)
+                                    target.write(data)
+                    checksum_hex = checksum.hexdigest()
         except BlockingIOError:
             raise StorageError("Another upload to this key is in progress")
 
@@ -1349,7 +1767,7 @@ class ObjectStorage:
         )
 
         stat = destination.stat()
-        etag = checksum.hexdigest()
+        etag = checksum_hex
         metadata = manifest.get("metadata")
 
         internal_meta = {"__etag__": etag, "__size__": str(stat.st_size)}
@@ -1522,21 +1940,41 @@ class ObjectStorage:
         return list(self._build_object_cache(bucket_path).keys())
 
     def _build_object_cache(self, bucket_path: Path) -> Dict[str, ObjectMeta]:
-        """Build a complete object metadata cache for a bucket.
-        
-        Uses os.scandir for fast directory walking and a persistent etag index.
-        """
         from concurrent.futures import ThreadPoolExecutor
-        
+
         bucket_id = bucket_path.name
         objects: Dict[str, ObjectMeta] = {}
         bucket_str = str(bucket_path)
         bucket_len = len(bucket_str) + 1
-        
+
+        if _HAS_RUST:
+            etag_index_path = self._system_bucket_root(bucket_id) / "etag_index.json"
+            raw = _rc.build_object_cache(
+                bucket_str,
+                str(self._bucket_meta_root(bucket_id)),
+                str(etag_index_path),
+            )
+            if raw["etag_cache_changed"] and raw["etag_cache"]:
+                try:
+                    etag_index_path.parent.mkdir(parents=True, exist_ok=True)
+                    with open(etag_index_path, 'w', encoding='utf-8') as f:
+                        json.dump(raw["etag_cache"], f)
+                except OSError:
+                    pass
+            for key, size, mtime, etag in raw["objects"]:
+                objects[key] = ObjectMeta(
+                    key=key,
+                    size=size,
+                    last_modified=datetime.fromtimestamp(mtime, timezone.utc),
+                    etag=etag,
+                    metadata=None,
+                )
+            return objects
+
         etag_index_path = self._system_bucket_root(bucket_id) / "etag_index.json"
         meta_cache: Dict[str, str] = {}
         index_mtime: float = 0
-        
+
         if etag_index_path.exists():
             try:
                 index_mtime = etag_index_path.stat().st_mtime
@@ -1544,10 +1982,10 @@ class ObjectStorage:
                     meta_cache = json.load(f)
             except (OSError, json.JSONDecodeError):
                 meta_cache = {}
-        
+
         meta_root = self._bucket_meta_root(bucket_id)
         needs_rebuild = False
-        
+
         if meta_root.exists() and index_mtime > 0:
             def check_newer(dir_path: str) -> bool:
                 try:
@@ -1565,7 +2003,7 @@ class ObjectStorage:
             needs_rebuild = check_newer(str(meta_root))
         elif not meta_cache:
             needs_rebuild = True
-            
+
         if needs_rebuild and meta_root.exists():
             meta_str = str(meta_root)
             meta_len = len(meta_str) + 1
@@ -1646,7 +2084,7 @@ class ObjectStorage:
                         json.dump(meta_cache, f)
                 except OSError:
                     pass
-        
+
         def scan_dir(dir_path: str) -> None:
             try:
                 with os.scandir(dir_path) as it:
@@ -1661,11 +2099,11 @@ class ObjectStorage:
                             first_part = rel.split(os.sep)[0] if os.sep in rel else rel
                             if first_part in self.INTERNAL_FOLDERS:
                                 continue
-                            
+
                             key = rel.replace(os.sep, '/')
                             try:
                                 stat = entry.stat()
-                                
+
                                 etag = meta_cache.get(key)
 
                                 objects[key] = ObjectMeta(
@@ -1673,13 +2111,13 @@ class ObjectStorage:
                                     size=stat.st_size,
                                     last_modified=datetime.fromtimestamp(stat.st_mtime, timezone.utc),
                                     etag=etag,
-                                    metadata=None, 
+                                    metadata=None,
                                 )
                             except OSError:
                                 pass
             except OSError:
                 pass
-        
+
         scan_dir(bucket_str)
         return objects
 
@@ -1763,11 +2201,6 @@ class ObjectStorage:
             return 0
 
     def _update_object_cache_entry(self, bucket_id: str, key: str, meta: Optional[ObjectMeta]) -> None:
-        """Update a single entry in the object cache instead of invalidating the whole cache.
-
-        This is a performance optimization - lazy update instead of full invalidation.
-        Cross-process invalidation is handled by checking stats.json mtime.
-        """
         with self._cache_lock:
             cached = self._object_cache.get(bucket_id)
             if cached:
@@ -1779,6 +2212,24 @@ class ObjectStorage:
             self._cache_version[bucket_id] = self._cache_version.get(bucket_id, 0) + 1
             self._sorted_key_cache.pop(bucket_id, None)
 
+        self._update_etag_index(bucket_id, key, meta.etag if meta else None)
+
+    def _update_etag_index(self, bucket_id: str, key: str, etag: Optional[str]) -> None:
+        etag_index_path = self._system_bucket_root(bucket_id) / "etag_index.json"
+        if not etag_index_path.exists():
+            return
+        try:
+            with open(etag_index_path, 'r', encoding='utf-8') as f:
+                index = json.load(f)
+            if etag is None:
+                index.pop(key, None)
+            else:
+                index[key] = etag
+            with open(etag_index_path, 'w', encoding='utf-8') as f:
+                json.dump(index, f)
+        except (OSError, json.JSONDecodeError):
+            pass
+
     def warm_cache(self, bucket_names: Optional[List[str]] = None) -> None:
         """Pre-warm the object cache for specified buckets or all buckets.
 
@@ -1829,30 +2280,40 @@ class ObjectStorage:
 
     def _read_bucket_config(self, bucket_name: str) -> dict[str, Any]:
         now = time.time()
+        config_path = self._bucket_config_path(bucket_name)
         cached = self._bucket_config_cache.get(bucket_name)
         if cached:
-            config, cached_time = cached
+            config, cached_time, cached_mtime = cached
             if now - cached_time < self._bucket_config_cache_ttl:
-                return config.copy()
+                try:
+                    current_mtime = config_path.stat().st_mtime if config_path.exists() else 0.0
+                except OSError:
+                    current_mtime = 0.0
+                if current_mtime == cached_mtime:
+                    return config.copy()
 
-        config_path = self._bucket_config_path(bucket_name)
         if not config_path.exists():
-            self._bucket_config_cache[bucket_name] = ({}, now)
+            self._bucket_config_cache[bucket_name] = ({}, now, 0.0)
             return {}
         try:
             data = json.loads(config_path.read_text(encoding="utf-8"))
             config = data if isinstance(data, dict) else {}
-            self._bucket_config_cache[bucket_name] = (config, now)
+            mtime = config_path.stat().st_mtime
+            self._bucket_config_cache[bucket_name] = (config, now, mtime)
             return config.copy()
         except (OSError, json.JSONDecodeError):
-            self._bucket_config_cache[bucket_name] = ({}, now)
+            self._bucket_config_cache[bucket_name] = ({}, now, 0.0)
             return {}
 
     def _write_bucket_config(self, bucket_name: str, payload: dict[str, Any]) -> None:
         config_path = self._bucket_config_path(bucket_name)
         config_path.parent.mkdir(parents=True, exist_ok=True)
         config_path.write_text(json.dumps(payload), encoding="utf-8")
-        self._bucket_config_cache[bucket_name] = (payload.copy(), time.time())
+        try:
+            mtime = config_path.stat().st_mtime
+        except OSError:
+            mtime = 0.0
+        self._bucket_config_cache[bucket_name] = (payload.copy(), time.time(), mtime)
 
     def _set_bucket_config_entry(self, bucket_name: str, key: str, value: Any | None) -> None:
         config = self._read_bucket_config(bucket_name)
@@ -1904,48 +2365,82 @@ class ObjectStorage:
             return self._meta_index_locks[index_path]
 
     def _read_index_entry(self, bucket_name: str, key: Path) -> Optional[Dict[str, Any]]:
+        cache_key = (bucket_name, str(key))
+        with self._cache_lock:
+            hit = self._meta_read_cache.get(cache_key)
+            if hit is not None:
+                self._meta_read_cache.move_to_end(cache_key)
+                cached = hit[0]
+                return copy.deepcopy(cached) if cached is not None else None
+
         index_path, entry_name = self._index_file_for_key(bucket_name, key)
-        if not index_path.exists():
-            return None
-        try:
-            index_data = json.loads(index_path.read_text(encoding="utf-8"))
-            return index_data.get(entry_name)
-        except (OSError, json.JSONDecodeError):
-            return None
+        if _HAS_RUST:
+            result = _rc.read_index_entry(str(index_path), entry_name)
+        else:
+            if not index_path.exists():
+                result = None
+            else:
+                try:
+                    index_data = json.loads(index_path.read_text(encoding="utf-8"))
+                    result = index_data.get(entry_name)
+                except (OSError, json.JSONDecodeError):
+                    result = None
+
+        with self._cache_lock:
+            while len(self._meta_read_cache) >= self._meta_read_cache_max:
+                self._meta_read_cache.popitem(last=False)
+            self._meta_read_cache[cache_key] = (copy.deepcopy(result) if result is not None else None,)
+
+        return result
+
+    def _invalidate_meta_read_cache(self, bucket_name: str, key: Path) -> None:
+        cache_key = (bucket_name, str(key))
+        with self._cache_lock:
+            self._meta_read_cache.pop(cache_key, None)
 
     def _write_index_entry(self, bucket_name: str, key: Path, entry: Dict[str, Any]) -> None:
         index_path, entry_name = self._index_file_for_key(bucket_name, key)
         lock = self._get_meta_index_lock(str(index_path))
         with lock:
-            index_path.parent.mkdir(parents=True, exist_ok=True)
-            index_data: Dict[str, Any] = {}
-            if index_path.exists():
-                try:
-                    index_data = json.loads(index_path.read_text(encoding="utf-8"))
-                except (OSError, json.JSONDecodeError):
-                    pass
-            index_data[entry_name] = entry
-            index_path.write_text(json.dumps(index_data), encoding="utf-8")
+            if _HAS_RUST:
+                _rc.write_index_entry(str(index_path), entry_name, json.dumps(entry))
+            else:
+                index_path.parent.mkdir(parents=True, exist_ok=True)
+                index_data: Dict[str, Any] = {}
+                if index_path.exists():
+                    try:
+                        index_data = json.loads(index_path.read_text(encoding="utf-8"))
+                    except (OSError, json.JSONDecodeError):
+                        pass
+                index_data[entry_name] = entry
+                index_path.write_text(json.dumps(index_data), encoding="utf-8")
+        self._invalidate_meta_read_cache(bucket_name, key)
 
     def _delete_index_entry(self, bucket_name: str, key: Path) -> None:
         index_path, entry_name = self._index_file_for_key(bucket_name, key)
         if not index_path.exists():
+            self._invalidate_meta_read_cache(bucket_name, key)
             return
         lock = self._get_meta_index_lock(str(index_path))
         with lock:
-            try:
-                index_data = json.loads(index_path.read_text(encoding="utf-8"))
-            except (OSError, json.JSONDecodeError):
-                return
-            if entry_name in index_data:
-                del index_data[entry_name]
-                if index_data:
-                    index_path.write_text(json.dumps(index_data), encoding="utf-8")
-                else:
-                    try:
-                        index_path.unlink()
-                    except OSError:
-                        pass
+            if _HAS_RUST:
+                _rc.delete_index_entry(str(index_path), entry_name)
+            else:
+                try:
+                    index_data = json.loads(index_path.read_text(encoding="utf-8"))
+                except (OSError, json.JSONDecodeError):
+                    self._invalidate_meta_read_cache(bucket_name, key)
+                    return
+                if entry_name in index_data:
+                    del index_data[entry_name]
+                    if index_data:
+                        index_path.write_text(json.dumps(index_data), encoding="utf-8")
+                    else:
+                        try:
+                            index_path.unlink()
+                        except OSError:
+                            pass
+        self._invalidate_meta_read_cache(bucket_name, key)
 
     def _normalize_metadata(self, metadata: Optional[Dict[str, str]]) -> Optional[Dict[str, str]]:
         if not metadata:
@@ -2042,15 +2537,24 @@ class ObjectStorage:
                 continue
 
     def _check_bucket_contents(self, bucket_path: Path) -> tuple[bool, bool, bool]:
-        """Check bucket for objects, versions, and multipart uploads in a single pass.
+        bucket_name = bucket_path.name
+
+        if _HAS_RUST:
+            return _rc.check_bucket_contents(
+                str(bucket_path),
+                [
+                    str(self._bucket_versions_root(bucket_name)),
+                    str(self._legacy_versions_root(bucket_name)),
+                ],
+                [
+                    str(self._multipart_bucket_root(bucket_name)),
+                    str(self._legacy_multipart_bucket_root(bucket_name)),
+                ],
+            )
 
-        Returns (has_visible_objects, has_archived_versions, has_active_multipart_uploads).
-        Uses early exit when all three are found.
-        """
         has_objects = False
         has_versions = False
         has_multipart = False
-        bucket_name = bucket_path.name
 
         for path in bucket_path.rglob("*"):
             if has_objects:

app/ui.py

@@ -508,11 +508,15 @@ def bucket_detail(bucket_name: str):
     can_manage_quota = is_replication_admin
 
     website_config = None
+    website_domains = []
     if website_hosting_enabled:
         try:
             website_config = storage.get_bucket_website(bucket_name)
         except StorageError:
             website_config = None
+        domain_store = current_app.extensions.get("website_domains")
+        if domain_store:
+            website_domains = domain_store.get_domains_for_bucket(bucket_name)
 
     objects_api_url = url_for("ui.list_bucket_objects", bucket_name=bucket_name)
     objects_stream_url = url_for("ui.stream_bucket_objects", bucket_name=bucket_name)
@@ -558,6 +562,7 @@ def bucket_detail(bucket_name: str):
         site_sync_enabled=site_sync_enabled,
         website_hosting_enabled=website_hosting_enabled,
         website_config=website_config,
+        website_domains=website_domains,
         can_manage_website=can_edit_policy,
     )
 
@@ -611,6 +616,7 @@ def stream_bucket_objects(bucket_name: str):
         return jsonify({"error": str(exc)}), 403
 
     prefix = request.args.get("prefix") or None
+    delimiter = request.args.get("delimiter") or None
 
     try:
         client = get_session_s3_client()
@@ -624,6 +630,7 @@ def stream_bucket_objects(bucket_name: str):
     return Response(
         stream_objects_ndjson(
             client, bucket_name, prefix, url_templates, display_tz, versioning_enabled,
+            delimiter=delimiter,
         ),
         mimetype='application/x-ndjson',
         headers={
@@ -634,6 +641,33 @@ def stream_bucket_objects(bucket_name: str):
     )
 
 
+@ui_bp.get("/buckets/<bucket_name>/objects/search")
+@limiter.limit("30 per minute")
+def search_bucket_objects(bucket_name: str):
+    principal = _current_principal()
+    try:
+        _authorize_ui(principal, bucket_name, "list")
+    except IamError as exc:
+        return jsonify({"error": str(exc)}), 403
+
+    query = request.args.get("q", "").strip()
+    if not query:
+        return jsonify({"results": [], "truncated": False})
+
+    try:
+        limit = max(1, min(int(request.args.get("limit", 500)), 1000))
+    except (ValueError, TypeError):
+        limit = 500
+
+    prefix = request.args.get("prefix", "").strip()
+
+    storage = _storage()
+    try:
+        return jsonify(storage.search_objects(bucket_name, query, prefix=prefix, limit=limit))
+    except StorageError as exc:
+        return jsonify({"error": str(exc)}), 404
+
+
 @ui_bp.post("/buckets/<bucket_name>/upload")
 @limiter.limit("30 per minute")
 def upload_object(bucket_name: str):
@@ -738,7 +772,6 @@ def initiate_multipart_upload(bucket_name: str):
 
 
 @ui_bp.put("/buckets/<bucket_name>/multipart/<upload_id>/parts")
-@limiter.exempt
 @csrf.exempt
 def upload_multipart_part(bucket_name: str, upload_id: str):
     principal = _current_principal()
@@ -1297,12 +1330,14 @@ def object_versions(bucket_name: str, object_key: str):
         for v in resp.get("Versions", []):
             if v.get("Key") != object_key:
                 continue
+            if v.get("IsLatest", False):
+                continue
             versions.append({
                 "version_id": v.get("VersionId", ""),
                 "last_modified": v["LastModified"].isoformat() if v.get("LastModified") else None,
                 "size": v.get("Size", 0),
                 "etag": v.get("ETag", "").strip('"'),
-                "is_latest": v.get("IsLatest", False),
+                "is_latest": False,
             })
         return jsonify({"versions": versions})
     except (ClientError, EndpointConnectionError, ConnectionClosedError) as exc:
@@ -2374,7 +2409,10 @@ def website_domains_dashboard():
     store = current_app.extensions.get("website_domains")
     mappings = store.list_all() if store else []
     storage = _storage()
-    buckets = [b.name for b in storage.list_buckets()]
+    buckets = [
+        b.name for b in storage.list_buckets()
+        if storage.get_bucket_website(b.name)
+    ]
     return render_template(
         "website_domains.html",
         mappings=mappings,
@@ -3293,9 +3331,12 @@ def sites_dashboard():
 @ui_bp.post("/sites/local")
 def update_local_site():
     principal = _current_principal()
+    wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest"
     try:
         _iam().authorize(principal, None, "iam:*")
     except IamError:
+        if wants_json:
+            return jsonify({"error": "Access denied"}), 403
         flash("Access denied", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3306,6 +3347,8 @@ def update_local_site():
     display_name = request.form.get("display_name", "").strip()
 
     if not site_id:
+        if wants_json:
+            return jsonify({"error": "Site ID is required"}), 400
         flash("Site ID is required", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3327,6 +3370,8 @@ def update_local_site():
     )
     registry.set_local_site(site)
 
+    if wants_json:
+        return jsonify({"message": "Local site configuration updated"})
     flash("Local site configuration updated", "success")
     return redirect(url_for("ui.sites_dashboard"))
 
@@ -3334,9 +3379,12 @@ def update_local_site():
 @ui_bp.post("/sites/peers")
 def add_peer_site():
     principal = _current_principal()
+    wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest"
     try:
         _iam().authorize(principal, None, "iam:*")
     except IamError:
+        if wants_json:
+            return jsonify({"error": "Access denied"}), 403
         flash("Access denied", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3348,9 +3396,13 @@ def add_peer_site():
     connection_id = request.form.get("connection_id", "").strip() or None
 
     if not site_id:
+        if wants_json:
+            return jsonify({"error": "Site ID is required"}), 400
         flash("Site ID is required", "danger")
         return redirect(url_for("ui.sites_dashboard"))
     if not endpoint:
+        if wants_json:
+            return jsonify({"error": "Endpoint is required"}), 400
         flash("Endpoint is required", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3362,10 +3414,14 @@ def add_peer_site():
     registry = _site_registry()
 
     if registry.get_peer(site_id):
+        if wants_json:
+            return jsonify({"error": f"Peer site '{site_id}' already exists"}), 409
         flash(f"Peer site '{site_id}' already exists", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
     if connection_id and not _connections().get(connection_id):
+        if wants_json:
+            return jsonify({"error": f"Connection '{connection_id}' not found"}), 404
         flash(f"Connection '{connection_id}' not found", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3379,6 +3435,11 @@ def add_peer_site():
     )
     registry.add_peer(peer)
 
+    if wants_json:
+        redirect_url = None
+        if connection_id:
+            redirect_url = url_for("ui.replication_wizard", site_id=site_id)
+        return jsonify({"message": f"Peer site '{site_id}' added", "redirect": redirect_url})
     flash(f"Peer site '{site_id}' added", "success")
 
     if connection_id:
@@ -3389,9 +3450,12 @@ def add_peer_site():
 @ui_bp.post("/sites/peers/<site_id>/update")
 def update_peer_site(site_id: str):
     principal = _current_principal()
+    wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest"
     try:
         _iam().authorize(principal, None, "iam:*")
     except IamError:
+        if wants_json:
+            return jsonify({"error": "Access denied"}), 403
         flash("Access denied", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3399,6 +3463,8 @@ def update_peer_site(site_id: str):
     existing = registry.get_peer(site_id)
 
     if not existing:
+        if wants_json:
+            return jsonify({"error": f"Peer site '{site_id}' not found"}), 404
         flash(f"Peer site '{site_id}' not found", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3406,7 +3472,10 @@ def update_peer_site(site_id: str):
     region = request.form.get("region", existing.region).strip()
     priority = request.form.get("priority", str(existing.priority))
     display_name = request.form.get("display_name", existing.display_name).strip()
-    connection_id = request.form.get("connection_id", "").strip() or existing.connection_id
+    if "connection_id" in request.form:
+        connection_id = request.form["connection_id"].strip() or None
+    else:
+        connection_id = existing.connection_id
 
     try:
         priority_int = int(priority)
@@ -3414,6 +3483,8 @@ def update_peer_site(site_id: str):
         priority_int = existing.priority
 
     if connection_id and not _connections().get(connection_id):
+        if wants_json:
+            return jsonify({"error": f"Connection '{connection_id}' not found"}), 404
         flash(f"Connection '{connection_id}' not found", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
@@ -3430,6 +3501,8 @@ def update_peer_site(site_id: str):
     )
     registry.update_peer(peer)
 
+    if wants_json:
+        return jsonify({"message": f"Peer site '{site_id}' updated"})
     flash(f"Peer site '{site_id}' updated", "success")
     return redirect(url_for("ui.sites_dashboard"))
 
@@ -3437,16 +3510,23 @@ def update_peer_site(site_id: str):
 @ui_bp.post("/sites/peers/<site_id>/delete")
 def delete_peer_site(site_id: str):
     principal = _current_principal()
+    wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest"
     try:
         _iam().authorize(principal, None, "iam:*")
     except IamError:
+        if wants_json:
+            return jsonify({"error": "Access denied"}), 403
         flash("Access denied", "danger")
         return redirect(url_for("ui.sites_dashboard"))
 
     registry = _site_registry()
     if registry.delete_peer(site_id):
+        if wants_json:
+            return jsonify({"message": f"Peer site '{site_id}' deleted"})
         flash(f"Peer site '{site_id}' deleted", "success")
     else:
+        if wants_json:
+            return jsonify({"error": f"Peer site '{site_id}' not found"}), 404
         flash(f"Peer site '{site_id}' not found", "danger")
 
     return redirect(url_for("ui.sites_dashboard"))

app/version.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-APP_VERSION = "0.3.0"
+APP_VERSION = "0.3.4"
 
 
 def get_version() -> str:

app/website_domains.py

@@ -35,13 +35,16 @@ class WebsiteDomainStore:
         self.config_path = config_path
         self._lock = threading.Lock()
         self._domains: Dict[str, str] = {}
+        self._last_mtime: float = 0.0
         self.reload()
 
     def reload(self) -> None:
         if not self.config_path.exists():
             self._domains = {}
+            self._last_mtime = 0.0
             return
         try:
+            self._last_mtime = self.config_path.stat().st_mtime
             with open(self.config_path, "r", encoding="utf-8") as f:
                 data = json.load(f)
                 if isinstance(data, dict):
@@ -51,19 +54,45 @@ class WebsiteDomainStore:
         except (OSError, json.JSONDecodeError):
             self._domains = {}
 
+    def _maybe_reload(self) -> None:
+        try:
+            if self.config_path.exists():
+                mtime = self.config_path.stat().st_mtime
+                if mtime != self._last_mtime:
+                    self._last_mtime = mtime
+                    with open(self.config_path, "r", encoding="utf-8") as f:
+                        data = json.load(f)
+                    if isinstance(data, dict):
+                        self._domains = {k.lower(): v for k, v in data.items()}
+                    else:
+                        self._domains = {}
+            elif self._domains:
+                self._domains = {}
+                self._last_mtime = 0.0
+        except (OSError, json.JSONDecodeError):
+            pass
+
     def _save(self) -> None:
         self.config_path.parent.mkdir(parents=True, exist_ok=True)
         with open(self.config_path, "w", encoding="utf-8") as f:
             json.dump(self._domains, f, indent=2)
+        self._last_mtime = self.config_path.stat().st_mtime
 
     def list_all(self) -> List[Dict[str, str]]:
         with self._lock:
+            self._maybe_reload()
             return [{"domain": d, "bucket": b} for d, b in self._domains.items()]
 
     def get_bucket(self, domain: str) -> Optional[str]:
         with self._lock:
+            self._maybe_reload()
             return self._domains.get(domain.lower())
 
+    def get_domains_for_bucket(self, bucket: str) -> List[str]:
+        with self._lock:
+            self._maybe_reload()
+            return [d for d, b in self._domains.items() if b == bucket]
+
     def set_mapping(self, domain: str, bucket: str) -> None:
         with self._lock:
             self._domains[domain.lower()] = bucket

docs.md

@@ -139,6 +139,7 @@ All configuration is done via environment variables. The table below lists every
 | `API_BASE_URL` | `http://127.0.0.1:5000` | Internal S3 API URL used by the web UI proxy. Also used for presigned URL generation. Set to your public URL if running behind a reverse proxy. |
 | `AWS_REGION` | `us-east-1` | Region embedded in SigV4 credential scope. |
 | `AWS_SERVICE` | `s3` | Service string for SigV4. |
+| `DISPLAY_TIMEZONE` | `UTC` | Timezone for timestamps in the web UI (e.g., `US/Eastern`, `Asia/Tokyo`). |
 
 ### IAM & Security
 
@@ -170,6 +171,7 @@ All configuration is done via environment variables. The table below lists every
 | `RATE_LIMIT_BUCKET_OPS` | `120 per minute` | Rate limit for bucket operations (PUT/DELETE/GET/POST on `/<bucket>`). |
 | `RATE_LIMIT_OBJECT_OPS` | `240 per minute` | Rate limit for object operations (PUT/GET/DELETE/POST on `/<bucket>/<key>`). |
 | `RATE_LIMIT_HEAD_OPS` | `100 per minute` | Rate limit for HEAD requests (bucket and object). |
+| `RATE_LIMIT_ADMIN` | `60 per minute` | Rate limit for admin API endpoints (`/admin/*`). |
 | `RATE_LIMIT_STORAGE_URI` | `memory://` | Storage backend for rate limits. Use `redis://host:port` for distributed setups. |
 
 ### Server Configuration
@@ -256,6 +258,12 @@ Once enabled, configure lifecycle rules via:
 | `MULTIPART_MIN_PART_SIZE` | `5242880` (5 MB) | Minimum part size for multipart uploads. |
 | `BUCKET_STATS_CACHE_TTL` | `60` | Seconds to cache bucket statistics. |
 | `BULK_DELETE_MAX_KEYS` | `500` | Maximum keys per bulk delete request. |
+| `BULK_DOWNLOAD_MAX_BYTES` | `1073741824` (1 GiB) | Maximum total size for bulk ZIP downloads. |
+| `OBJECT_CACHE_TTL` | `60` | Seconds to cache object metadata. |
+
+#### Gzip Compression
+
+API responses for JSON, XML, HTML, CSS, and JavaScript are automatically gzip-compressed when the client sends `Accept-Encoding: gzip`. Compression activates for responses larger than 500 bytes and is handled by a WSGI middleware (`app/compression.py`). Binary object downloads and streaming responses are never compressed. No configuration is needed.
 
 ### Server Settings
 
@@ -285,6 +293,12 @@ If running behind a reverse proxy (e.g., Nginx, Cloudflare, or a tunnel), ensure
 
 The application automatically trusts these headers to generate correct presigned URLs (e.g., `https://s3.example.com/...` instead of `http://127.0.0.1:5000/...`). Alternatively, you can explicitly set `API_BASE_URL` to your public endpoint.
 
+| Variable | Default | Notes |
+| --- | --- | --- |
+| `NUM_TRUSTED_PROXIES` | `1` | Number of trusted reverse proxies for `X-Forwarded-*` header processing. |
+| `ALLOWED_REDIRECT_HOSTS` | `""` | Comma-separated whitelist of safe redirect targets. Empty allows only same-host redirects. |
+| `ALLOW_INTERNAL_ENDPOINTS` | `false` | Allow connections to internal/private IPs for webhooks and replication targets. **Keep disabled in production unless needed.** |
+
 ## 4. Upgrading and Updates
 
 ### Version Checking
@@ -912,7 +926,7 @@ Objects with forward slashes (`/`) in their keys are displayed as a folder hiera
 
 - Select multiple objects using checkboxes
 - **Bulk Delete**: Delete multiple objects at once
-- **Bulk Download**: Download selected objects as individual files
+- **Bulk Download**: Download selected objects as a single ZIP archive (up to `BULK_DOWNLOAD_MAX_BYTES`, default 1 GiB)
 
 #### Search & Filter
 
@@ -985,6 +999,7 @@ MyFSIO supports **server-side encryption at rest** to protect your data. When en
 |------|-------------|
 | **AES-256 (SSE-S3)** | Server-managed encryption using a local master key |
 | **KMS (SSE-KMS)** | Encryption using customer-managed keys via the built-in KMS |
+| **SSE-C** | Server-side encryption with customer-provided keys (per-request) |
 
 ### Enabling Encryption
 
@@ -1083,6 +1098,44 @@ encrypted, metadata = ClientEncryptionHelper.encrypt_for_upload(plaintext, key)
 decrypted = ClientEncryptionHelper.decrypt_from_download(encrypted, metadata, key)
 ```
 
+### SSE-C (Customer-Provided Keys)
+
+With SSE-C, you provide your own 256-bit AES encryption key with each request. The server encrypts/decrypts using your key but never stores it. You must supply the same key for both upload and download.
+
+**Required headers:**
+
+| Header | Value |
+|--------|-------|
+| `x-amz-server-side-encryption-customer-algorithm` | `AES256` |
+| `x-amz-server-side-encryption-customer-key` | Base64-encoded 256-bit key |
+| `x-amz-server-side-encryption-customer-key-MD5` | Base64-encoded MD5 of the key |
+
+```bash
+# Generate a 256-bit key
+KEY=$(openssl rand -base64 32)
+KEY_MD5=$(echo -n "$KEY" | base64 -d | openssl dgst -md5 -binary | base64)
+
+# Upload with SSE-C
+curl -X PUT "http://localhost:5000/my-bucket/secret.txt" \
+  -H "X-Access-Key: ..." -H "X-Secret-Key: ..." \
+  -H "x-amz-server-side-encryption-customer-algorithm: AES256" \
+  -H "x-amz-server-side-encryption-customer-key: $KEY" \
+  -H "x-amz-server-side-encryption-customer-key-MD5: $KEY_MD5" \
+  --data-binary @secret.txt
+
+# Download with SSE-C (same key required)
+curl "http://localhost:5000/my-bucket/secret.txt" \
+  -H "X-Access-Key: ..." -H "X-Secret-Key: ..." \
+  -H "x-amz-server-side-encryption-customer-algorithm: AES256" \
+  -H "x-amz-server-side-encryption-customer-key: $KEY" \
+  -H "x-amz-server-side-encryption-customer-key-MD5: $KEY_MD5"
+```
+
+**Key points:**
+- SSE-C does not require `ENCRYPTION_ENABLED` or `KMS_ENABLED` — the key is provided per-request
+- If you lose your key, the data is irrecoverable
+- The MD5 header is optional but recommended for integrity verification
+
 ### Important Notes
 
 - **Existing objects are NOT encrypted** - Only new uploads after enabling encryption are encrypted
@@ -1959,6 +2012,20 @@ curl -X PUT "http://localhost:5000/my-bucket/file.txt" \
   -H "x-amz-meta-newkey: newvalue"
 ```
 
+### MoveObject (UI)
+
+Move an object to a different key or bucket. This is a UI-only convenience operation that performs a copy followed by a delete of the source. Requires `read` and `delete` on the source, and `write` on the destination.
+
+```bash
+# Move via UI API
+curl -X POST "http://localhost:5100/ui/buckets/my-bucket/objects/old-path/file.txt/move" \
+  -H "Content-Type: application/json" \
+  --cookie "session=..." \
+  -d '{"dest_bucket": "other-bucket", "dest_key": "new-path/file.txt"}'
+```
+
+The move is atomic from the caller's perspective: if the copy succeeds but the delete fails, the object exists in both locations (no data loss).
+
 ### UploadPartCopy
 
 Copy data from an existing object into a multipart upload part:

myfsio_core/Cargo.lock

@@ -1,421 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 4
-
-[[package]]
-name = "aho-corasick"
-version = "1.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "allocator-api2"
-version = "0.2.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
-
-[[package]]
-name = "bitflags"
-version = "2.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
-
-[[package]]
-name = "block-buffer"
-version = "0.10.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
-dependencies = [
- "generic-array",
-]
-
-[[package]]
-name = "cfg-if"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
-
-[[package]]
-name = "cpufeatures"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "crypto-common"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
-dependencies = [
- "generic-array",
- "typenum",
-]
-
-[[package]]
-name = "digest"
-version = "0.10.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
-dependencies = [
- "block-buffer",
- "crypto-common",
- "subtle",
-]
-
-[[package]]
-name = "equivalent"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
-
-[[package]]
-name = "foldhash"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
-
-[[package]]
-name = "generic-array"
-version = "0.14.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
-dependencies = [
- "typenum",
- "version_check",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.15.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
-dependencies = [
- "allocator-api2",
- "equivalent",
- "foldhash",
-]
-
-[[package]]
-name = "heck"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
-
-[[package]]
-name = "hex"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
-
-[[package]]
-name = "hmac"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
-dependencies = [
- "digest",
-]
-
-[[package]]
-name = "libc"
-version = "0.2.182"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
-
-[[package]]
-name = "lock_api"
-version = "0.4.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
-dependencies = [
- "scopeguard",
-]
-
-[[package]]
-name = "lru"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f8cc7106155f10bdf99a6f379688f543ad6596a415375b36a59a054ceda1198"
-dependencies = [
- "hashbrown",
-]
-
-[[package]]
-name = "md-5"
-version = "0.10.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
-dependencies = [
- "cfg-if",
- "digest",
-]
-
-[[package]]
-name = "memchr"
-version = "2.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
-
-[[package]]
-name = "myfsio_core"
-version = "0.1.0"
-dependencies = [
- "hex",
- "hmac",
- "lru",
- "md-5",
- "parking_lot",
- "pyo3",
- "regex",
- "sha2",
- "unicode-normalization",
-]
-
-[[package]]
-name = "once_cell"
-version = "1.21.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
-
-[[package]]
-name = "parking_lot"
-version = "0.12.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
-dependencies = [
- "lock_api",
- "parking_lot_core",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.9.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
-dependencies = [
- "cfg-if",
- "libc",
- "redox_syscall",
- "smallvec",
- "windows-link",
-]
-
-[[package]]
-name = "portable-atomic"
-version = "1.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.106"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "pyo3"
-version = "0.28.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14c738662e2181be11cb82487628404254902bb3225d8e9e99c31f3ef82a405c"
-dependencies = [
- "libc",
- "once_cell",
- "portable-atomic",
- "pyo3-build-config",
- "pyo3-ffi",
- "pyo3-macros",
-]
-
-[[package]]
-name = "pyo3-build-config"
-version = "0.28.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9ca0864a7dd3c133a7f3f020cbff2e12e88420da854c35540fd20ce2d60e435"
-dependencies = [
- "target-lexicon",
-]
-
-[[package]]
-name = "pyo3-ffi"
-version = "0.28.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9dfc1956b709823164763a34cc42bbfd26b8730afa77809a3df8b94a3ae3b059"
-dependencies = [
- "libc",
- "pyo3-build-config",
-]
-
-[[package]]
-name = "pyo3-macros"
-version = "0.28.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29dc660ad948bae134d579661d08033fbb1918f4529c3bbe3257a68f2009ddf2"
-dependencies = [
- "proc-macro2",
- "pyo3-macros-backend",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "pyo3-macros-backend"
-version = "0.28.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e78cd6c6d718acfcedf26c3d21fe0f053624368b0d44298c55d7138fde9331f7"
-dependencies = [
- "heck",
- "proc-macro2",
- "pyo3-build-config",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.44"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.5.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
-dependencies = [
- "bitflags",
-]
-
-[[package]]
-name = "regex"
-version = "1.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-automata",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.4.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-syntax"
-version = "0.8.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c"
-
-[[package]]
-name = "scopeguard"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
-
-[[package]]
-name = "sha2"
-version = "0.10.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
-dependencies = [
- "cfg-if",
- "cpufeatures",
- "digest",
-]
-
-[[package]]
-name = "smallvec"
-version = "1.15.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
-
-[[package]]
-name = "subtle"
-version = "2.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
-
-[[package]]
-name = "syn"
-version = "2.0.116"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3df424c70518695237746f84cede799c9c58fcb37450d7b23716568cc8bc69cb"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "target-lexicon"
-version = "0.13.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
-
-[[package]]
-name = "tinyvec"
-version = "1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa"
-dependencies = [
- "tinyvec_macros",
-]
-
-[[package]]
-name = "tinyvec_macros"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
-
-[[package]]
-name = "typenum"
-version = "1.19.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
-
-[[package]]
-name = "unicode-normalization"
-version = "0.1.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
-dependencies = [
- "tinyvec",
-]
-
-[[package]]
-name = "version_check"
-version = "0.9.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
-
-[[package]]
-name = "windows-link"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"

myfsio_core/Cargo.toml

@@ -14,6 +14,11 @@ sha2 = "0.10"
 md-5 = "0.10"
 hex = "0.4"
 unicode-normalization = "0.1"
+serde_json = "1"
 regex = "1"
 lru = "0.14"
 parking_lot = "0.12"
+percent-encoding = "2"
+aes-gcm = "0.10"
+hkdf = "0.12"
+uuid = { version = "1", features = ["v4"] }

myfsio_core/src/crypto.rs

@@ -0,0 +1,192 @@
+use aes_gcm::aead::Aead;
+use aes_gcm::{Aes256Gcm, KeyInit, Nonce};
+use hkdf::Hkdf;
+use pyo3::exceptions::{PyIOError, PyValueError};
+use pyo3::prelude::*;
+use sha2::Sha256;
+use std::fs::File;
+use std::io::{Read, Seek, SeekFrom, Write};
+
+const DEFAULT_CHUNK_SIZE: usize = 65536;
+const HEADER_SIZE: usize = 4;
+
+fn read_exact_chunk(reader: &mut impl Read, buf: &mut [u8]) -> std::io::Result<usize> {
+    let mut filled = 0;
+    while filled < buf.len() {
+        match reader.read(&mut buf[filled..]) {
+            Ok(0) => break,
+            Ok(n) => filled += n,
+            Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
+            Err(e) => return Err(e),
+        }
+    }
+    Ok(filled)
+}
+
+fn derive_chunk_nonce(base_nonce: &[u8], chunk_index: u32) -> Result<[u8; 12], String> {
+    let hkdf = Hkdf::<Sha256>::new(Some(base_nonce), b"chunk_nonce");
+    let mut okm = [0u8; 12];
+    hkdf.expand(&chunk_index.to_be_bytes(), &mut okm)
+        .map_err(|e| format!("HKDF expand failed: {}", e))?;
+    Ok(okm)
+}
+
+#[pyfunction]
+#[pyo3(signature = (input_path, output_path, key, base_nonce, chunk_size=DEFAULT_CHUNK_SIZE))]
+pub fn encrypt_stream_chunked(
+    py: Python<'_>,
+    input_path: &str,
+    output_path: &str,
+    key: &[u8],
+    base_nonce: &[u8],
+    chunk_size: usize,
+) -> PyResult<u32> {
+    if key.len() != 32 {
+        return Err(PyValueError::new_err(format!(
+            "Key must be 32 bytes, got {}",
+            key.len()
+        )));
+    }
+    if base_nonce.len() != 12 {
+        return Err(PyValueError::new_err(format!(
+            "Base nonce must be 12 bytes, got {}",
+            base_nonce.len()
+        )));
+    }
+
+    let chunk_size = if chunk_size == 0 {
+        DEFAULT_CHUNK_SIZE
+    } else {
+        chunk_size
+    };
+
+    let inp = input_path.to_owned();
+    let out = output_path.to_owned();
+    let key_arr: [u8; 32] = key.try_into().unwrap();
+    let nonce_arr: [u8; 12] = base_nonce.try_into().unwrap();
+
+    py.detach(move || {
+        let cipher = Aes256Gcm::new(&key_arr.into());
+
+        let mut infile = File::open(&inp)
+            .map_err(|e| PyIOError::new_err(format!("Failed to open input: {}", e)))?;
+        let mut outfile = File::create(&out)
+            .map_err(|e| PyIOError::new_err(format!("Failed to create output: {}", e)))?;
+
+        outfile
+            .write_all(&[0u8; 4])
+            .map_err(|e| PyIOError::new_err(format!("Failed to write header: {}", e)))?;
+
+        let mut buf = vec![0u8; chunk_size];
+        let mut chunk_index: u32 = 0;
+
+        loop {
+            let n = read_exact_chunk(&mut infile, &mut buf)
+                .map_err(|e| PyIOError::new_err(format!("Failed to read: {}", e)))?;
+            if n == 0 {
+                break;
+            }
+
+            let nonce_bytes = derive_chunk_nonce(&nonce_arr, chunk_index)
+                .map_err(|e| PyValueError::new_err(e))?;
+            let nonce = Nonce::from_slice(&nonce_bytes);
+
+            let encrypted = cipher
+                .encrypt(nonce, &buf[..n])
+                .map_err(|e| PyValueError::new_err(format!("Encrypt failed: {}", e)))?;
+
+            let size = encrypted.len() as u32;
+            outfile
+                .write_all(&size.to_be_bytes())
+                .map_err(|e| PyIOError::new_err(format!("Failed to write chunk size: {}", e)))?;
+            outfile
+                .write_all(&encrypted)
+                .map_err(|e| PyIOError::new_err(format!("Failed to write chunk: {}", e)))?;
+
+            chunk_index += 1;
+        }
+
+        outfile
+            .seek(SeekFrom::Start(0))
+            .map_err(|e| PyIOError::new_err(format!("Failed to seek: {}", e)))?;
+        outfile
+            .write_all(&chunk_index.to_be_bytes())
+            .map_err(|e| PyIOError::new_err(format!("Failed to write chunk count: {}", e)))?;
+
+        Ok(chunk_index)
+    })
+}
+
+#[pyfunction]
+pub fn decrypt_stream_chunked(
+    py: Python<'_>,
+    input_path: &str,
+    output_path: &str,
+    key: &[u8],
+    base_nonce: &[u8],
+) -> PyResult<u32> {
+    if key.len() != 32 {
+        return Err(PyValueError::new_err(format!(
+            "Key must be 32 bytes, got {}",
+            key.len()
+        )));
+    }
+    if base_nonce.len() != 12 {
+        return Err(PyValueError::new_err(format!(
+            "Base nonce must be 12 bytes, got {}",
+            base_nonce.len()
+        )));
+    }
+
+    let inp = input_path.to_owned();
+    let out = output_path.to_owned();
+    let key_arr: [u8; 32] = key.try_into().unwrap();
+    let nonce_arr: [u8; 12] = base_nonce.try_into().unwrap();
+
+    py.detach(move || {
+        let cipher = Aes256Gcm::new(&key_arr.into());
+
+        let mut infile = File::open(&inp)
+            .map_err(|e| PyIOError::new_err(format!("Failed to open input: {}", e)))?;
+        let mut outfile = File::create(&out)
+            .map_err(|e| PyIOError::new_err(format!("Failed to create output: {}", e)))?;
+
+        let mut header = [0u8; HEADER_SIZE];
+        infile
+            .read_exact(&mut header)
+            .map_err(|e| PyIOError::new_err(format!("Failed to read header: {}", e)))?;
+        let chunk_count = u32::from_be_bytes(header);
+
+        let mut size_buf = [0u8; HEADER_SIZE];
+        for chunk_index in 0..chunk_count {
+            infile
+                .read_exact(&mut size_buf)
+                .map_err(|e| {
+                    PyIOError::new_err(format!(
+                        "Failed to read chunk {} size: {}",
+                        chunk_index, e
+                    ))
+                })?;
+            let chunk_size = u32::from_be_bytes(size_buf) as usize;
+
+            let mut encrypted = vec![0u8; chunk_size];
+            infile.read_exact(&mut encrypted).map_err(|e| {
+                PyIOError::new_err(format!("Failed to read chunk {}: {}", chunk_index, e))
+            })?;
+
+            let nonce_bytes = derive_chunk_nonce(&nonce_arr, chunk_index)
+                .map_err(|e| PyValueError::new_err(e))?;
+            let nonce = Nonce::from_slice(&nonce_bytes);
+
+            let decrypted = cipher.decrypt(nonce, encrypted.as_ref()).map_err(|e| {
+                PyValueError::new_err(format!("Decrypt chunk {} failed: {}", chunk_index, e))
+            })?;
+
+            outfile.write_all(&decrypted).map_err(|e| {
+                PyIOError::new_err(format!("Failed to write chunk {}: {}", chunk_index, e))
+            })?;
+        }
+
+        Ok(chunk_count)
+    })
+}

myfsio_core/src/lib.rs

@@ -1,5 +1,9 @@
+mod crypto;
 mod hashing;
+mod metadata;
 mod sigv4;
+mod storage;
+mod streaming;
 mod validation;
 
 use pyo3::prelude::*;
@@ -10,6 +14,7 @@ mod myfsio_core {
 
     #[pymodule_init]
     fn init(m: &Bound<'_, PyModule>) -> PyResult<()> {
+        m.add_function(wrap_pyfunction!(sigv4::verify_sigv4_signature, m)?)?;
         m.add_function(wrap_pyfunction!(sigv4::derive_signing_key, m)?)?;
         m.add_function(wrap_pyfunction!(sigv4::compute_signature, m)?)?;
         m.add_function(wrap_pyfunction!(sigv4::build_string_to_sign, m)?)?;
@@ -25,6 +30,22 @@ mod myfsio_core {
         m.add_function(wrap_pyfunction!(validation::validate_object_key, m)?)?;
         m.add_function(wrap_pyfunction!(validation::validate_bucket_name, m)?)?;
 
+        m.add_function(wrap_pyfunction!(metadata::read_index_entry, m)?)?;
+
+        m.add_function(wrap_pyfunction!(storage::write_index_entry, m)?)?;
+        m.add_function(wrap_pyfunction!(storage::delete_index_entry, m)?)?;
+        m.add_function(wrap_pyfunction!(storage::check_bucket_contents, m)?)?;
+        m.add_function(wrap_pyfunction!(storage::shallow_scan, m)?)?;
+        m.add_function(wrap_pyfunction!(storage::bucket_stats_scan, m)?)?;
+        m.add_function(wrap_pyfunction!(storage::search_objects_scan, m)?)?;
+        m.add_function(wrap_pyfunction!(storage::build_object_cache, m)?)?;
+
+        m.add_function(wrap_pyfunction!(streaming::stream_to_file_with_md5, m)?)?;
+        m.add_function(wrap_pyfunction!(streaming::assemble_parts_with_md5, m)?)?;
+
+        m.add_function(wrap_pyfunction!(crypto::encrypt_stream_chunked, m)?)?;
+        m.add_function(wrap_pyfunction!(crypto::decrypt_stream_chunked, m)?)?;
+
         Ok(())
     }
 }

myfsio_core/src/metadata.rs

@@ -0,0 +1,71 @@
+use pyo3::exceptions::PyValueError;
+use pyo3::prelude::*;
+use pyo3::types::{PyDict, PyList, PyString};
+use serde_json::Value;
+use std::fs;
+
+const MAX_DEPTH: u32 = 64;
+
+fn value_to_py(py: Python<'_>, v: &Value, depth: u32) -> PyResult<Py<PyAny>> {
+    if depth > MAX_DEPTH {
+        return Err(PyValueError::new_err("JSON nesting too deep"));
+    }
+    match v {
+        Value::Null => Ok(py.None()),
+        Value::Bool(b) => Ok((*b).into_pyobject(py)?.to_owned().into_any().unbind()),
+        Value::Number(n) => {
+            if let Some(i) = n.as_i64() {
+                Ok(i.into_pyobject(py)?.into_any().unbind())
+            } else if let Some(f) = n.as_f64() {
+                Ok(f.into_pyobject(py)?.into_any().unbind())
+            } else {
+                Ok(py.None())
+            }
+        }
+        Value::String(s) => Ok(PyString::new(py, s).into_any().unbind()),
+        Value::Array(arr) => {
+            let list = PyList::empty(py);
+            for item in arr {
+                list.append(value_to_py(py, item, depth + 1)?)?;
+            }
+            Ok(list.into_any().unbind())
+        }
+        Value::Object(map) => {
+            let dict = PyDict::new(py);
+            for (k, val) in map {
+                dict.set_item(k, value_to_py(py, val, depth + 1)?)?;
+            }
+            Ok(dict.into_any().unbind())
+        }
+    }
+}
+
+#[pyfunction]
+pub fn read_index_entry(
+    py: Python<'_>,
+    path: &str,
+    entry_name: &str,
+) -> PyResult<Option<Py<PyAny>>> {
+    let path_owned = path.to_owned();
+    let entry_owned = entry_name.to_owned();
+
+    let entry: Option<Value> = py.detach(move || -> PyResult<Option<Value>> {
+        let content = match fs::read_to_string(&path_owned) {
+            Ok(c) => c,
+            Err(_) => return Ok(None),
+        };
+        let parsed: Value = match serde_json::from_str(&content) {
+            Ok(v) => v,
+            Err(_) => return Ok(None),
+        };
+        match parsed {
+            Value::Object(mut map) => Ok(map.remove(&entry_owned)),
+            _ => Ok(None),
+        }
+    })?;
+
+    match entry {
+        Some(val) => Ok(Some(value_to_py(py, &val, 0)?)),
+        None => Ok(None),
+    }
+}

myfsio_core/src/sigv4.rs

@@ -1,6 +1,7 @@
 use hmac::{Hmac, Mac};
 use lru::LruCache;
 use parking_lot::Mutex;
+use percent_encoding::{percent_encode, AsciiSet, NON_ALPHANUMERIC};
 use pyo3::prelude::*;
 use sha2::{Digest, Sha256};
 use std::num::NonZeroUsize;
@@ -19,14 +20,29 @@ static SIGNING_KEY_CACHE: LazyLock<Mutex<LruCache<(String, String, String, Strin
 
 const CACHE_TTL_SECS: u64 = 60;
 
+const AWS_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC
+    .remove(b'-')
+    .remove(b'_')
+    .remove(b'.')
+    .remove(b'~');
+
 fn hmac_sha256(key: &[u8], msg: &[u8]) -> Vec<u8> {
     let mut mac = HmacSha256::new_from_slice(key).expect("HMAC key length is always valid");
     mac.update(msg);
     mac.finalize().into_bytes().to_vec()
 }
 
-#[pyfunction]
-pub fn derive_signing_key(
+fn sha256_hex(data: &[u8]) -> String {
+    let mut hasher = Sha256::new();
+    hasher.update(data);
+    hex::encode(hasher.finalize())
+}
+
+fn aws_uri_encode(input: &str) -> String {
+    percent_encode(input.as_bytes(), AWS_ENCODE_SET).to_string()
+}
+
+fn derive_signing_key_cached(
     secret_key: &str,
     date_stamp: &str,
     region: &str,
@@ -68,18 +84,91 @@ pub fn derive_signing_key(
     k_signing
 }
 
+fn constant_time_compare_inner(a: &[u8], b: &[u8]) -> bool {
+    if a.len() != b.len() {
+        return false;
+    }
+    let mut result: u8 = 0;
+    for (x, y) in a.iter().zip(b.iter()) {
+        result |= x ^ y;
+    }
+    result == 0
+}
+
+#[pyfunction]
+pub fn verify_sigv4_signature(
+    method: &str,
+    canonical_uri: &str,
+    query_params: Vec<(String, String)>,
+    signed_headers_str: &str,
+    header_values: Vec<(String, String)>,
+    payload_hash: &str,
+    amz_date: &str,
+    date_stamp: &str,
+    region: &str,
+    service: &str,
+    secret_key: &str,
+    provided_signature: &str,
+) -> bool {
+    let mut sorted_params = query_params;
+    sorted_params.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)));
+
+    let canonical_query_string = sorted_params
+        .iter()
+        .map(|(k, v)| format!("{}={}", aws_uri_encode(k), aws_uri_encode(v)))
+        .collect::<Vec<_>>()
+        .join("&");
+
+    let mut canonical_headers = String::new();
+    for (name, value) in &header_values {
+        let lower_name = name.to_lowercase();
+        let normalized = value.split_whitespace().collect::<Vec<_>>().join(" ");
+        let final_value = if lower_name == "expect" && normalized.is_empty() {
+            "100-continue"
+        } else {
+            &normalized
+        };
+        canonical_headers.push_str(&lower_name);
+        canonical_headers.push(':');
+        canonical_headers.push_str(final_value);
+        canonical_headers.push('\n');
+    }
+
+    let canonical_request = format!(
+        "{}\n{}\n{}\n{}\n{}\n{}",
+        method, canonical_uri, canonical_query_string, canonical_headers, signed_headers_str, payload_hash
+    );
+
+    let credential_scope = format!("{}/{}/{}/aws4_request", date_stamp, region, service);
+    let cr_hash = sha256_hex(canonical_request.as_bytes());
+    let string_to_sign = format!(
+        "AWS4-HMAC-SHA256\n{}\n{}\n{}",
+        amz_date, credential_scope, cr_hash
+    );
+
+    let signing_key = derive_signing_key_cached(secret_key, date_stamp, region, service);
+    let calculated = hmac_sha256(&signing_key, string_to_sign.as_bytes());
+    let calculated_hex = hex::encode(&calculated);
+
+    constant_time_compare_inner(calculated_hex.as_bytes(), provided_signature.as_bytes())
+}
+
+#[pyfunction]
+pub fn derive_signing_key(
+    secret_key: &str,
+    date_stamp: &str,
+    region: &str,
+    service: &str,
+) -> Vec<u8> {
+    derive_signing_key_cached(secret_key, date_stamp, region, service)
+}
+
 #[pyfunction]
 pub fn compute_signature(signing_key: &[u8], string_to_sign: &str) -> String {
     let sig = hmac_sha256(signing_key, string_to_sign.as_bytes());
     hex::encode(sig)
 }
 
-fn sha256_hex(data: &[u8]) -> String {
-    let mut hasher = Sha256::new();
-    hasher.update(data);
-    hex::encode(hasher.finalize())
-}
-
 #[pyfunction]
 pub fn build_string_to_sign(
     amz_date: &str,
@@ -87,19 +176,15 @@ pub fn build_string_to_sign(
     canonical_request: &str,
 ) -> String {
     let cr_hash = sha256_hex(canonical_request.as_bytes());
-    format!("AWS4-HMAC-SHA256\n{}\n{}\n{}", amz_date, credential_scope, cr_hash)
+    format!(
+        "AWS4-HMAC-SHA256\n{}\n{}\n{}",
+        amz_date, credential_scope, cr_hash
+    )
 }
 
 #[pyfunction]
 pub fn constant_time_compare(a: &str, b: &str) -> bool {
-    if a.len() != b.len() {
-        return false;
-    }
-    let mut result: u8 = 0;
-    for (x, y) in a.bytes().zip(b.bytes()) {
-        result |= x ^ y;
-    }
-    result == 0
+    constant_time_compare_inner(a.as_bytes(), b.as_bytes())
 }
 
 #[pyfunction]

myfsio_core/src/storage.rs

@@ -0,0 +1,817 @@
+use pyo3::exceptions::PyIOError;
+use pyo3::prelude::*;
+use pyo3::types::{PyDict, PyList, PyString, PyTuple};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::fs;
+use std::path::Path;
+use std::time::SystemTime;
+
+const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"];
+
+fn system_time_to_epoch(t: SystemTime) -> f64 {
+    t.duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_secs_f64())
+        .unwrap_or(0.0)
+}
+
+fn extract_etag_from_meta_bytes(content: &[u8]) -> Option<String> {
+    let marker = b"\"__etag__\"";
+    let idx = content.windows(marker.len()).position(|w| w == marker)?;
+    let after = &content[idx + marker.len()..];
+    let start = after.iter().position(|&b| b == b'"')? + 1;
+    let rest = &after[start..];
+    let end = rest.iter().position(|&b| b == b'"')?;
+    std::str::from_utf8(&rest[..end]).ok().map(|s| s.to_owned())
+}
+
+fn has_any_file(root: &str) -> bool {
+    let root_path = Path::new(root);
+    if !root_path.is_dir() {
+        return false;
+    }
+    let mut stack = vec![root_path.to_path_buf()];
+    while let Some(current) = stack.pop() {
+        let entries = match fs::read_dir(&current) {
+            Ok(e) => e,
+            Err(_) => continue,
+        };
+        for entry_result in entries {
+            let entry = match entry_result {
+                Ok(e) => e,
+                Err(_) => continue,
+            };
+            let ft = match entry.file_type() {
+                Ok(ft) => ft,
+                Err(_) => continue,
+            };
+            if ft.is_file() {
+                return true;
+            }
+            if ft.is_dir() && !ft.is_symlink() {
+                stack.push(entry.path());
+            }
+        }
+    }
+    false
+}
+
+#[pyfunction]
+pub fn write_index_entry(
+    py: Python<'_>,
+    path: &str,
+    entry_name: &str,
+    entry_data_json: &str,
+) -> PyResult<()> {
+    let path_owned = path.to_owned();
+    let entry_owned = entry_name.to_owned();
+    let data_owned = entry_data_json.to_owned();
+
+    py.detach(move || -> PyResult<()> {
+        let entry_value: Value = serde_json::from_str(&data_owned)
+            .map_err(|e| PyIOError::new_err(format!("Failed to parse entry data: {}", e)))?;
+
+        if let Some(parent) = Path::new(&path_owned).parent() {
+            let _ = fs::create_dir_all(parent);
+        }
+
+        let mut index_data: serde_json::Map<String, Value> = match fs::read_to_string(&path_owned)
+        {
+            Ok(content) => serde_json::from_str(&content).unwrap_or_default(),
+            Err(_) => serde_json::Map::new(),
+        };
+
+        index_data.insert(entry_owned, entry_value);
+
+        let serialized = serde_json::to_string(&Value::Object(index_data))
+            .map_err(|e| PyIOError::new_err(format!("Failed to serialize index: {}", e)))?;
+
+        fs::write(&path_owned, serialized)
+            .map_err(|e| PyIOError::new_err(format!("Failed to write index: {}", e)))?;
+
+        Ok(())
+    })
+}
+
+#[pyfunction]
+pub fn delete_index_entry(py: Python<'_>, path: &str, entry_name: &str) -> PyResult<bool> {
+    let path_owned = path.to_owned();
+    let entry_owned = entry_name.to_owned();
+
+    py.detach(move || -> PyResult<bool> {
+        let content = match fs::read_to_string(&path_owned) {
+            Ok(c) => c,
+            Err(_) => return Ok(false),
+        };
+
+        let mut index_data: serde_json::Map<String, Value> =
+            match serde_json::from_str(&content) {
+                Ok(v) => v,
+                Err(_) => return Ok(false),
+            };
+
+        if index_data.remove(&entry_owned).is_none() {
+            return Ok(false);
+        }
+
+        if index_data.is_empty() {
+            let _ = fs::remove_file(&path_owned);
+            return Ok(true);
+        }
+
+        let serialized = serde_json::to_string(&Value::Object(index_data))
+            .map_err(|e| PyIOError::new_err(format!("Failed to serialize index: {}", e)))?;
+
+        fs::write(&path_owned, serialized)
+            .map_err(|e| PyIOError::new_err(format!("Failed to write index: {}", e)))?;
+
+        Ok(false)
+    })
+}
+
+#[pyfunction]
+pub fn check_bucket_contents(
+    py: Python<'_>,
+    bucket_path: &str,
+    version_roots: Vec<String>,
+    multipart_roots: Vec<String>,
+) -> PyResult<(bool, bool, bool)> {
+    let bucket_owned = bucket_path.to_owned();
+
+    py.detach(move || -> PyResult<(bool, bool, bool)> {
+        let mut has_objects = false;
+        let bucket_p = Path::new(&bucket_owned);
+        if bucket_p.is_dir() {
+            let mut stack = vec![bucket_p.to_path_buf()];
+            'obj_scan: while let Some(current) = stack.pop() {
+                let is_root = current == bucket_p;
+                let entries = match fs::read_dir(&current) {
+                    Ok(e) => e,
+                    Err(_) => continue,
+                };
+                for entry_result in entries {
+                    let entry = match entry_result {
+                        Ok(e) => e,
+                        Err(_) => continue,
+                    };
+                    let ft = match entry.file_type() {
+                        Ok(ft) => ft,
+                        Err(_) => continue,
+                    };
+                    if is_root {
+                        if let Some(name) = entry.file_name().to_str() {
+                            if INTERNAL_FOLDERS.contains(&name) {
+                                continue;
+                            }
+                        }
+                    }
+                    if ft.is_file() && !ft.is_symlink() {
+                        has_objects = true;
+                        break 'obj_scan;
+                    }
+                    if ft.is_dir() && !ft.is_symlink() {
+                        stack.push(entry.path());
+                    }
+                }
+            }
+        }
+
+        let mut has_versions = false;
+        for root in &version_roots {
+            if has_versions {
+                break;
+            }
+            has_versions = has_any_file(root);
+        }
+
+        let mut has_multipart = false;
+        for root in &multipart_roots {
+            if has_multipart {
+                break;
+            }
+            has_multipart = has_any_file(root);
+        }
+
+        Ok((has_objects, has_versions, has_multipart))
+    })
+}
+
+#[pyfunction]
+pub fn shallow_scan(
+    py: Python<'_>,
+    target_dir: &str,
+    prefix: &str,
+    meta_cache_json: &str,
+) -> PyResult<Py<PyAny>> {
+    let target_owned = target_dir.to_owned();
+    let prefix_owned = prefix.to_owned();
+    let cache_owned = meta_cache_json.to_owned();
+
+    let result: (
+        Vec<(String, u64, f64, Option<String>)>,
+        Vec<String>,
+        Vec<(String, bool)>,
+    ) = py.detach(move || -> PyResult<(
+        Vec<(String, u64, f64, Option<String>)>,
+        Vec<String>,
+        Vec<(String, bool)>,
+    )> {
+        let meta_cache: HashMap<String, String> =
+            serde_json::from_str(&cache_owned).unwrap_or_default();
+
+        let mut files: Vec<(String, u64, f64, Option<String>)> = Vec::new();
+        let mut dirs: Vec<String> = Vec::new();
+
+        let entries = match fs::read_dir(&target_owned) {
+            Ok(e) => e,
+            Err(_) => return Ok((files, dirs, Vec::new())),
+        };
+
+        for entry_result in entries {
+            let entry = match entry_result {
+                Ok(e) => e,
+                Err(_) => continue,
+            };
+            let name = match entry.file_name().into_string() {
+                Ok(n) => n,
+                Err(_) => continue,
+            };
+            if INTERNAL_FOLDERS.contains(&name.as_str()) {
+                continue;
+            }
+            let ft = match entry.file_type() {
+                Ok(ft) => ft,
+                Err(_) => continue,
+            };
+            if ft.is_dir() && !ft.is_symlink() {
+                let cp = format!("{}{}/", prefix_owned, name);
+                dirs.push(cp);
+            } else if ft.is_file() && !ft.is_symlink() {
+                let key = format!("{}{}", prefix_owned, name);
+                let md = match entry.metadata() {
+                    Ok(m) => m,
+                    Err(_) => continue,
+                };
+                let size = md.len();
+                let mtime = md
+                    .modified()
+                    .map(system_time_to_epoch)
+                    .unwrap_or(0.0);
+                let etag = meta_cache.get(&key).cloned();
+                files.push((key, size, mtime, etag));
+            }
+        }
+
+        files.sort_by(|a, b| a.0.cmp(&b.0));
+        dirs.sort();
+
+        let mut merged: Vec<(String, bool)> = Vec::with_capacity(files.len() + dirs.len());
+        let mut fi = 0;
+        let mut di = 0;
+        while fi < files.len() && di < dirs.len() {
+            if files[fi].0 <= dirs[di] {
+                merged.push((files[fi].0.clone(), false));
+                fi += 1;
+            } else {
+                merged.push((dirs[di].clone(), true));
+                di += 1;
+            }
+        }
+        while fi < files.len() {
+            merged.push((files[fi].0.clone(), false));
+            fi += 1;
+        }
+        while di < dirs.len() {
+            merged.push((dirs[di].clone(), true));
+            di += 1;
+        }
+
+        Ok((files, dirs, merged))
+    })?;
+
+    let (files, dirs, merged) = result;
+
+    let dict = PyDict::new(py);
+
+    let files_list = PyList::empty(py);
+    for (key, size, mtime, etag) in &files {
+        let etag_py: Py<PyAny> = match etag {
+            Some(e) => PyString::new(py, e).into_any().unbind(),
+            None => py.None(),
+        };
+        let tuple = PyTuple::new(py, &[
+            PyString::new(py, key).into_any().unbind(),
+            size.into_pyobject(py)?.into_any().unbind(),
+            mtime.into_pyobject(py)?.into_any().unbind(),
+            etag_py,
+        ])?;
+        files_list.append(tuple)?;
+    }
+    dict.set_item("files", files_list)?;
+
+    let dirs_list = PyList::empty(py);
+    for d in &dirs {
+        dirs_list.append(PyString::new(py, d))?;
+    }
+    dict.set_item("dirs", dirs_list)?;
+
+    let merged_list = PyList::empty(py);
+    for (key, is_dir) in &merged {
+        let bool_obj: Py<PyAny> = if *is_dir {
+            true.into_pyobject(py)?.to_owned().into_any().unbind()
+        } else {
+            false.into_pyobject(py)?.to_owned().into_any().unbind()
+        };
+        let tuple = PyTuple::new(py, &[
+            PyString::new(py, key).into_any().unbind(),
+            bool_obj,
+        ])?;
+        merged_list.append(tuple)?;
+    }
+    dict.set_item("merged_keys", merged_list)?;
+
+    Ok(dict.into_any().unbind())
+}
+
+#[pyfunction]
+pub fn bucket_stats_scan(
+    py: Python<'_>,
+    bucket_path: &str,
+    versions_root: &str,
+) -> PyResult<(u64, u64, u64, u64)> {
+    let bucket_owned = bucket_path.to_owned();
+    let versions_owned = versions_root.to_owned();
+
+    py.detach(move || -> PyResult<(u64, u64, u64, u64)> {
+        let mut object_count: u64 = 0;
+        let mut total_bytes: u64 = 0;
+
+        let bucket_p = Path::new(&bucket_owned);
+        if bucket_p.is_dir() {
+            let mut stack = vec![bucket_p.to_path_buf()];
+            while let Some(current) = stack.pop() {
+                let is_root = current == bucket_p;
+                let entries = match fs::read_dir(&current) {
+                    Ok(e) => e,
+                    Err(_) => continue,
+                };
+                for entry_result in entries {
+                    let entry = match entry_result {
+                        Ok(e) => e,
+                        Err(_) => continue,
+                    };
+                    if is_root {
+                        if let Some(name) = entry.file_name().to_str() {
+                            if INTERNAL_FOLDERS.contains(&name) {
+                                continue;
+                            }
+                        }
+                    }
+                    let ft = match entry.file_type() {
+                        Ok(ft) => ft,
+                        Err(_) => continue,
+                    };
+                    if ft.is_dir() && !ft.is_symlink() {
+                        stack.push(entry.path());
+                    } else if ft.is_file() && !ft.is_symlink() {
+                        object_count += 1;
+                        if let Ok(md) = entry.metadata() {
+                            total_bytes += md.len();
+                        }
+                    }
+                }
+            }
+        }
+
+        let mut version_count: u64 = 0;
+        let mut version_bytes: u64 = 0;
+
+        let versions_p = Path::new(&versions_owned);
+        if versions_p.is_dir() {
+            let mut stack = vec![versions_p.to_path_buf()];
+            while let Some(current) = stack.pop() {
+                let entries = match fs::read_dir(&current) {
+                    Ok(e) => e,
+                    Err(_) => continue,
+                };
+                for entry_result in entries {
+                    let entry = match entry_result {
+                        Ok(e) => e,
+                        Err(_) => continue,
+                    };
+                    let ft = match entry.file_type() {
+                        Ok(ft) => ft,
+                        Err(_) => continue,
+                    };
+                    if ft.is_dir() && !ft.is_symlink() {
+                        stack.push(entry.path());
+                    } else if ft.is_file() && !ft.is_symlink() {
+                        if let Some(name) = entry.file_name().to_str() {
+                            if name.ends_with(".bin") {
+                                version_count += 1;
+                                if let Ok(md) = entry.metadata() {
+                                    version_bytes += md.len();
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        Ok((object_count, total_bytes, version_count, version_bytes))
+    })
+}
+
+#[pyfunction]
+#[pyo3(signature = (bucket_path, search_root, query, limit))]
+pub fn search_objects_scan(
+    py: Python<'_>,
+    bucket_path: &str,
+    search_root: &str,
+    query: &str,
+    limit: usize,
+) -> PyResult<Py<PyAny>> {
+    let bucket_owned = bucket_path.to_owned();
+    let search_owned = search_root.to_owned();
+    let query_owned = query.to_owned();
+
+    let result: (Vec<(String, u64, f64)>, bool) = py.detach(
+        move || -> PyResult<(Vec<(String, u64, f64)>, bool)> {
+            let query_lower = query_owned.to_lowercase();
+            let bucket_len = bucket_owned.len() + 1;
+            let scan_limit = limit * 4;
+            let mut matched: usize = 0;
+            let mut results: Vec<(String, u64, f64)> = Vec::new();
+
+            let search_p = Path::new(&search_owned);
+            if !search_p.is_dir() {
+                return Ok((results, false));
+            }
+
+            let bucket_p = Path::new(&bucket_owned);
+            let mut stack = vec![search_p.to_path_buf()];
+
+            'scan: while let Some(current) = stack.pop() {
+                let is_bucket_root = current == bucket_p;
+                let entries = match fs::read_dir(&current) {
+                    Ok(e) => e,
+                    Err(_) => continue,
+                };
+                for entry_result in entries {
+                    let entry = match entry_result {
+                        Ok(e) => e,
+                        Err(_) => continue,
+                    };
+                    if is_bucket_root {
+                        if let Some(name) = entry.file_name().to_str() {
+                            if INTERNAL_FOLDERS.contains(&name) {
+                                continue;
+                            }
+                        }
+                    }
+                    let ft = match entry.file_type() {
+                        Ok(ft) => ft,
+                        Err(_) => continue,
+                    };
+                    if ft.is_dir() && !ft.is_symlink() {
+                        stack.push(entry.path());
+                    } else if ft.is_file() && !ft.is_symlink() {
+                        let full_path = entry.path();
+                        let full_str = full_path.to_string_lossy();
+                        if full_str.len() <= bucket_len {
+                            continue;
+                        }
+                        let key = full_str[bucket_len..].replace('\\', "/");
+                        if key.to_lowercase().contains(&query_lower) {
+                            if let Ok(md) = entry.metadata() {
+                                let size = md.len();
+                                let mtime = md
+                                    .modified()
+                                    .map(system_time_to_epoch)
+                                    .unwrap_or(0.0);
+                                results.push((key, size, mtime));
+                                matched += 1;
+                            }
+                        }
+                        if matched >= scan_limit {
+                            break 'scan;
+                        }
+                    }
+                }
+            }
+
+            results.sort_by(|a, b| a.0.cmp(&b.0));
+            let truncated = results.len() > limit;
+            results.truncate(limit);
+
+            Ok((results, truncated))
+        },
+    )?;
+
+    let (results, truncated) = result;
+
+    let dict = PyDict::new(py);
+
+    let results_list = PyList::empty(py);
+    for (key, size, mtime) in &results {
+        let tuple = PyTuple::new(py, &[
+            PyString::new(py, key).into_any().unbind(),
+            size.into_pyobject(py)?.into_any().unbind(),
+            mtime.into_pyobject(py)?.into_any().unbind(),
+        ])?;
+        results_list.append(tuple)?;
+    }
+    dict.set_item("results", results_list)?;
+    dict.set_item("truncated", truncated)?;
+
+    Ok(dict.into_any().unbind())
+}
+
+#[pyfunction]
+pub fn build_object_cache(
+    py: Python<'_>,
+    bucket_path: &str,
+    meta_root: &str,
+    etag_index_path: &str,
+) -> PyResult<Py<PyAny>> {
+    let bucket_owned = bucket_path.to_owned();
+    let meta_owned = meta_root.to_owned();
+    let index_path_owned = etag_index_path.to_owned();
+
+    let result: (HashMap<String, String>, Vec<(String, u64, f64, Option<String>)>, bool) =
+        py.detach(move || -> PyResult<(
+            HashMap<String, String>,
+            Vec<(String, u64, f64, Option<String>)>,
+            bool,
+        )> {
+            let mut meta_cache: HashMap<String, String> = HashMap::new();
+            let mut index_mtime: f64 = 0.0;
+            let mut etag_cache_changed = false;
+
+            let index_p = Path::new(&index_path_owned);
+            if index_p.is_file() {
+                if let Ok(md) = fs::metadata(&index_path_owned) {
+                    index_mtime = md
+                        .modified()
+                        .map(system_time_to_epoch)
+                        .unwrap_or(0.0);
+                }
+                if let Ok(content) = fs::read_to_string(&index_path_owned) {
+                    if let Ok(parsed) = serde_json::from_str::<HashMap<String, String>>(&content) {
+                        meta_cache = parsed;
+                    }
+                }
+            }
+
+            let meta_p = Path::new(&meta_owned);
+            let mut needs_rebuild = false;
+
+            if meta_p.is_dir() && index_mtime > 0.0 {
+                fn check_newer(dir: &Path, index_mtime: f64) -> bool {
+                    let entries = match fs::read_dir(dir) {
+                        Ok(e) => e,
+                        Err(_) => return false,
+                    };
+                    for entry_result in entries {
+                        let entry = match entry_result {
+                            Ok(e) => e,
+                            Err(_) => continue,
+                        };
+                        let ft = match entry.file_type() {
+                            Ok(ft) => ft,
+                            Err(_) => continue,
+                        };
+                        if ft.is_dir() && !ft.is_symlink() {
+                            if check_newer(&entry.path(), index_mtime) {
+                                return true;
+                            }
+                        } else if ft.is_file() {
+                            if let Some(name) = entry.file_name().to_str() {
+                                if name.ends_with(".meta.json") || name == "_index.json" {
+                                    if let Ok(md) = entry.metadata() {
+                                        let mt = md
+                                            .modified()
+                                            .map(system_time_to_epoch)
+                                            .unwrap_or(0.0);
+                                        if mt > index_mtime {
+                                            return true;
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    false
+                }
+                needs_rebuild = check_newer(meta_p, index_mtime);
+            } else if meta_cache.is_empty() {
+                needs_rebuild = true;
+            }
+
+            if needs_rebuild && meta_p.is_dir() {
+                let meta_str = meta_owned.clone();
+                let meta_len = meta_str.len() + 1;
+                let mut index_files: Vec<String> = Vec::new();
+                let mut legacy_meta_files: Vec<(String, String)> = Vec::new();
+
+                fn collect_meta(
+                    dir: &Path,
+                    meta_len: usize,
+                    index_files: &mut Vec<String>,
+                    legacy_meta_files: &mut Vec<(String, String)>,
+                ) {
+                    let entries = match fs::read_dir(dir) {
+                        Ok(e) => e,
+                        Err(_) => return,
+                    };
+                    for entry_result in entries {
+                        let entry = match entry_result {
+                            Ok(e) => e,
+                            Err(_) => continue,
+                        };
+                        let ft = match entry.file_type() {
+                            Ok(ft) => ft,
+                            Err(_) => continue,
+                        };
+                        if ft.is_dir() && !ft.is_symlink() {
+                            collect_meta(&entry.path(), meta_len, index_files, legacy_meta_files);
+                        } else if ft.is_file() {
+                            if let Some(name) = entry.file_name().to_str() {
+                                let full = entry.path().to_string_lossy().to_string();
+                                if name == "_index.json" {
+                                    index_files.push(full);
+                                } else if name.ends_with(".meta.json") {
+                                    if full.len() > meta_len {
+                                        let rel = &full[meta_len..];
+                                        let key = if rel.len() > 10 {
+                                            rel[..rel.len() - 10].replace('\\', "/")
+                                        } else {
+                                            continue;
+                                        };
+                                        legacy_meta_files.push((key, full));
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+
+                collect_meta(
+                    meta_p,
+                    meta_len,
+                    &mut index_files,
+                    &mut legacy_meta_files,
+                );
+
+                meta_cache.clear();
+
+                for idx_path in &index_files {
+                    if let Ok(content) = fs::read_to_string(idx_path) {
+                        if let Ok(idx_data) = serde_json::from_str::<HashMap<String, Value>>(&content) {
+                            let rel_dir = if idx_path.len() > meta_len {
+                                let r = &idx_path[meta_len..];
+                                r.replace('\\', "/")
+                            } else {
+                                String::new()
+                            };
+                            let dir_prefix = if rel_dir.ends_with("/_index.json") {
+                                &rel_dir[..rel_dir.len() - "/_index.json".len()]
+                            } else {
+                                ""
+                            };
+                            for (entry_name, entry_data) in &idx_data {
+                                let key = if dir_prefix.is_empty() {
+                                    entry_name.clone()
+                                } else {
+                                    format!("{}/{}", dir_prefix, entry_name)
+                                };
+                                if let Some(meta_obj) = entry_data.get("metadata") {
+                                    if let Some(etag) = meta_obj.get("__etag__") {
+                                        if let Some(etag_str) = etag.as_str() {
+                                            meta_cache.insert(key, etag_str.to_owned());
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+
+                for (key, path) in &legacy_meta_files {
+                    if meta_cache.contains_key(key) {
+                        continue;
+                    }
+                    if let Ok(content) = fs::read(path) {
+                        if let Some(etag) = extract_etag_from_meta_bytes(&content) {
+                            meta_cache.insert(key.clone(), etag);
+                        }
+                    }
+                }
+
+                etag_cache_changed = true;
+            }
+
+            let bucket_p = Path::new(&bucket_owned);
+            let bucket_len = bucket_owned.len() + 1;
+            let mut objects: Vec<(String, u64, f64, Option<String>)> = Vec::new();
+
+            if bucket_p.is_dir() {
+                let mut stack = vec![bucket_p.to_path_buf()];
+                while let Some(current) = stack.pop() {
+                    let entries = match fs::read_dir(&current) {
+                        Ok(e) => e,
+                        Err(_) => continue,
+                    };
+                    for entry_result in entries {
+                        let entry = match entry_result {
+                            Ok(e) => e,
+                            Err(_) => continue,
+                        };
+                        let ft = match entry.file_type() {
+                            Ok(ft) => ft,
+                            Err(_) => continue,
+                        };
+                        if ft.is_dir() && !ft.is_symlink() {
+                            let full = entry.path();
+                            let full_str = full.to_string_lossy();
+                            if full_str.len() > bucket_len {
+                                let first_part: &str = if let Some(sep_pos) =
+                                    full_str[bucket_len..].find(|c: char| c == '\\' || c == '/')
+                                {
+                                    &full_str[bucket_len..bucket_len + sep_pos]
+                                } else {
+                                    &full_str[bucket_len..]
+                                };
+                                if INTERNAL_FOLDERS.contains(&first_part) {
+                                    continue;
+                                }
+                            } else if let Some(name) = entry.file_name().to_str() {
+                                if INTERNAL_FOLDERS.contains(&name) {
+                                    continue;
+                                }
+                            }
+                            stack.push(full);
+                        } else if ft.is_file() && !ft.is_symlink() {
+                            let full = entry.path();
+                            let full_str = full.to_string_lossy();
+                            if full_str.len() <= bucket_len {
+                                continue;
+                            }
+                            let rel = &full_str[bucket_len..];
+                            let first_part: &str =
+                                if let Some(sep_pos) = rel.find(|c: char| c == '\\' || c == '/') {
+                                    &rel[..sep_pos]
+                                } else {
+                                    rel
+                                };
+                            if INTERNAL_FOLDERS.contains(&first_part) {
+                                continue;
+                            }
+                            let key = rel.replace('\\', "/");
+                            if let Ok(md) = entry.metadata() {
+                                let size = md.len();
+                                let mtime = md
+                                    .modified()
+                                    .map(system_time_to_epoch)
+                                    .unwrap_or(0.0);
+                                let etag = meta_cache.get(&key).cloned();
+                                objects.push((key, size, mtime, etag));
+                            }
+                        }
+                    }
+                }
+            }
+
+            Ok((meta_cache, objects, etag_cache_changed))
+        })?;
+
+    let (meta_cache, objects, etag_cache_changed) = result;
+
+    let dict = PyDict::new(py);
+
+    let cache_dict = PyDict::new(py);
+    for (k, v) in &meta_cache {
+        cache_dict.set_item(k, v)?;
+    }
+    dict.set_item("etag_cache", cache_dict)?;
+
+    let objects_list = PyList::empty(py);
+    for (key, size, mtime, etag) in &objects {
+        let etag_py: Py<PyAny> = match etag {
+            Some(e) => PyString::new(py, e).into_any().unbind(),
+            None => py.None(),
+        };
+        let tuple = PyTuple::new(py, &[
+            PyString::new(py, key).into_any().unbind(),
+            size.into_pyobject(py)?.into_any().unbind(),
+            mtime.into_pyobject(py)?.into_any().unbind(),
+            etag_py,
+        ])?;
+        objects_list.append(tuple)?;
+    }
+    dict.set_item("objects", objects_list)?;
+    dict.set_item("etag_cache_changed", etag_cache_changed)?;
+
+    Ok(dict.into_any().unbind())
+}

myfsio_core/src/streaming.rs

@@ -0,0 +1,107 @@
+use md5::{Digest, Md5};
+use pyo3::exceptions::{PyIOError, PyValueError};
+use pyo3::prelude::*;
+use std::fs::{self, File};
+use std::io::{Read, Write};
+use uuid::Uuid;
+
+const DEFAULT_CHUNK_SIZE: usize = 262144;
+
+#[pyfunction]
+#[pyo3(signature = (stream, tmp_dir, chunk_size=DEFAULT_CHUNK_SIZE))]
+pub fn stream_to_file_with_md5(
+    py: Python<'_>,
+    stream: &Bound<'_, PyAny>,
+    tmp_dir: &str,
+    chunk_size: usize,
+) -> PyResult<(String, String, u64)> {
+    let chunk_size = if chunk_size == 0 {
+        DEFAULT_CHUNK_SIZE
+    } else {
+        chunk_size
+    };
+
+    fs::create_dir_all(tmp_dir)
+        .map_err(|e| PyIOError::new_err(format!("Failed to create tmp dir: {}", e)))?;
+
+    let tmp_name = format!("{}.tmp", Uuid::new_v4().as_hyphenated());
+    let tmp_path_buf = std::path::PathBuf::from(tmp_dir).join(&tmp_name);
+    let tmp_path = tmp_path_buf.to_string_lossy().into_owned();
+
+    let mut file = File::create(&tmp_path)
+        .map_err(|e| PyIOError::new_err(format!("Failed to create temp file: {}", e)))?;
+    let mut hasher = Md5::new();
+    let mut total_bytes: u64 = 0;
+
+    let result: PyResult<()> = (|| {
+        loop {
+            let chunk: Vec<u8> = stream.call_method1("read", (chunk_size,))?.extract()?;
+            if chunk.is_empty() {
+                break;
+            }
+            hasher.update(&chunk);
+            file.write_all(&chunk)
+                .map_err(|e| PyIOError::new_err(format!("Failed to write: {}", e)))?;
+            total_bytes += chunk.len() as u64;
+
+            py.check_signals()?;
+        }
+        Ok(())
+    })();
+
+    if let Err(e) = result {
+        drop(file);
+        let _ = fs::remove_file(&tmp_path);
+        return Err(e);
+    }
+
+    drop(file);
+
+    let md5_hex = format!("{:x}", hasher.finalize());
+    Ok((tmp_path, md5_hex, total_bytes))
+}
+
+#[pyfunction]
+pub fn assemble_parts_with_md5(
+    py: Python<'_>,
+    part_paths: Vec<String>,
+    dest_path: &str,
+) -> PyResult<String> {
+    if part_paths.is_empty() {
+        return Err(PyValueError::new_err("No parts to assemble"));
+    }
+
+    let dest = dest_path.to_owned();
+    let parts = part_paths;
+
+    py.detach(move || {
+        if let Some(parent) = std::path::Path::new(&dest).parent() {
+            fs::create_dir_all(parent)
+                .map_err(|e| PyIOError::new_err(format!("Failed to create dest dir: {}", e)))?;
+        }
+
+        let mut target = File::create(&dest)
+            .map_err(|e| PyIOError::new_err(format!("Failed to create dest file: {}", e)))?;
+        let mut hasher = Md5::new();
+        let mut buf = vec![0u8; 1024 * 1024];
+
+        for part_path in &parts {
+            let mut part = File::open(part_path)
+                .map_err(|e| PyIOError::new_err(format!("Failed to open part {}: {}", part_path, e)))?;
+            loop {
+                let n = part
+                    .read(&mut buf)
+                    .map_err(|e| PyIOError::new_err(format!("Failed to read part: {}", e)))?;
+                if n == 0 {
+                    break;
+                }
+                hasher.update(&buf[..n]);
+                target
+                    .write_all(&buf[..n])
+                    .map_err(|e| PyIOError::new_err(format!("Failed to write: {}", e)))?;
+            }
+        }
+
+        Ok(format!("{:x}", hasher.finalize()))
+    })
+}

myfsio_core/target/.rustc_info.json

@@ -1 +0,0 @@
-{"rustc_fingerprint":13172970000770725120,"outputs":{"7971740275564407648":{"success":true,"status":"","code":0,"stdout":"___.exe\nlib___.rlib\n___.dll\n___.dll\n___.lib\n___.dll\nC:\\Users\\jun\\.rustup\\toolchains\\stable-x86_64-pc-windows-msvc\npacked\n___\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_abi=\"\"\ntarget_arch=\"x86_64\"\ntarget_endian=\"little\"\ntarget_env=\"msvc\"\ntarget_family=\"windows\"\ntarget_feature=\"cmpxchg16b\"\ntarget_feature=\"fxsr\"\ntarget_feature=\"sse\"\ntarget_feature=\"sse2\"\ntarget_feature=\"sse3\"\ntarget_has_atomic=\"128\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_os=\"windows\"\ntarget_pointer_width=\"64\"\ntarget_vendor=\"pc\"\nwindows\n","stderr":""},"17747080675513052775":{"success":true,"status":"","code":0,"stdout":"rustc 1.93.1 (01f6ddf75 2026-02-11)\nbinary: rustc\ncommit-hash: 01f6ddf7588f42ae2d7eb0a2f21d44e8e96674cf\ncommit-date: 2026-02-11\nhost: x86_64-pc-windows-msvc\nrelease: 1.93.1\nLLVM version: 21.1.8\n","stderr":""}},"successes":{}}

myfsio_core/target/CACHEDIR.TAG

@@ -1,3 +0,0 @@
-Signature: 8a477f597d28d172789f06886806bc55
-# This file is a cache directory tag created by cargo.
-# For information about cache directory tags see https://bford.info/cachedir/

myfsio_core/target/release/.fingerprint/aho-corasick-45694771b543be75/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/aho-corasick-45694771b543be75/lib-aho_corasick

@@ -1 +0,0 @@
-801af22cf202da8e

myfsio_core/target/release/.fingerprint/aho-corasick-45694771b543be75/lib-aho_corasick.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"perf-literal\", \"std\"]","declared_features":"[\"default\", \"logging\", \"perf-literal\", \"std\"]","target":7534583537114156500,"profile":2040997289075261528,"path":6364296192483896971,"deps":[[1363051979936526615,"memchr",false,11090220145123168660]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\aho-corasick-45694771b543be75\\dep-lib-aho_corasick","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/allocator-api2-db7934dbe96de5b4/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/allocator-api2-db7934dbe96de5b4/lib-allocator_api2

@@ -1 +0,0 @@
-435555ec2fb592e3

myfsio_core/target/release/.fingerprint/allocator-api2-db7934dbe96de5b4/lib-allocator_api2.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"alloc\"]","declared_features":"[\"alloc\", \"default\", \"fresh-rust\", \"nightly\", \"serde\", \"std\"]","target":5388200169723499962,"profile":4067574213046180398,"path":10654049299693593327,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\allocator-api2-db7934dbe96de5b4\\dep-lib-allocator_api2","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/autocfg-1c4fb7a37cc3df69/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/autocfg-1c4fb7a37cc3df69/lib-autocfg

@@ -1 +0,0 @@
-d28af275d001c358

myfsio_core/target/release/.fingerprint/autocfg-1c4fb7a37cc3df69/lib-autocfg.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":6962977057026645649,"profile":1369601567987815722,"path":9853093265219907461,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\autocfg-1c4fb7a37cc3df69\\dep-lib-autocfg","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/block-buffer-95b0ac364bec72f9/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/block-buffer-95b0ac364bec72f9/lib-block_buffer

@@ -1 +0,0 @@
-1fbf4ba9542edced

myfsio_core/target/release/.fingerprint/block-buffer-95b0ac364bec72f9/lib-block_buffer.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":4098124618827574291,"profile":2040997289075261528,"path":3658007358608479489,"deps":[[10520923840501062997,"generic_array",false,11555283918993371487]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\block-buffer-95b0ac364bec72f9\\dep-lib-block_buffer","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/cfg-if-be2711f84a777e73/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/cfg-if-be2711f84a777e73/lib-cfg_if

@@ -1 +0,0 @@
-37923e6f5f9687ab

myfsio_core/target/release/.fingerprint/cfg-if-be2711f84a777e73/lib-cfg_if.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[\"core\", \"rustc-dep-of-std\"]","target":13840298032947503755,"profile":2040997289075261528,"path":4093486168504982869,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\cfg-if-be2711f84a777e73\\dep-lib-cfg_if","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/cpufeatures-980094f8735c42d1/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/cpufeatures-980094f8735c42d1/lib-cpufeatures

@@ -1 +0,0 @@
-603e28136cf5763c

myfsio_core/target/release/.fingerprint/cpufeatures-980094f8735c42d1/lib-cpufeatures.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":2330704043955282025,"profile":2040997289075261528,"path":13200428550696548327,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\cpufeatures-980094f8735c42d1\\dep-lib-cpufeatures","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/crypto-common-289a508abdda3048/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/crypto-common-289a508abdda3048/lib-crypto_common

@@ -1 +0,0 @@
-896672d759b5299c

myfsio_core/target/release/.fingerprint/crypto-common-289a508abdda3048/lib-crypto_common.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"std\"]","declared_features":"[\"getrandom\", \"rand_core\", \"std\"]","target":12082577455412410174,"profile":2040997289075261528,"path":14902376638882023040,"deps":[[857979250431893282,"typenum",false,7416411392359930020],[10520923840501062997,"generic_array",false,11555283918993371487]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\crypto-common-289a508abdda3048\\dep-lib-crypto_common","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/digest-a91458bfa5613332/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/digest-a91458bfa5613332/lib-digest

@@ -1 +0,0 @@
-914a617b9f05c9d8

myfsio_core/target/release/.fingerprint/digest-a91458bfa5613332/lib-digest.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"alloc\", \"block-buffer\", \"core-api\", \"default\", \"mac\", \"std\", \"subtle\"]","declared_features":"[\"alloc\", \"blobby\", \"block-buffer\", \"const-oid\", \"core-api\", \"default\", \"dev\", \"mac\", \"oid\", \"rand_core\", \"std\", \"subtle\"]","target":7510122432137863311,"profile":2040997289075261528,"path":11503432597517024930,"deps":[[6039282458970808711,"crypto_common",false,11252724541433210505],[10626340395483396037,"block_buffer",false,17139625223017709343],[17003143334332120809,"subtle",false,8597342066671925934]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\digest-a91458bfa5613332\\dep-lib-digest","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/equivalent-943ac856871c0988/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/equivalent-943ac856871c0988/lib-equivalent

@@ -1 +0,0 @@
-3b95cf48bbd7dc53

myfsio_core/target/release/.fingerprint/equivalent-943ac856871c0988/lib-equivalent.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":1524667692659508025,"profile":2040997289075261528,"path":17534356223679657546,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\equivalent-943ac856871c0988\\dep-lib-equivalent","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/foldhash-b8a92f8c10d550f7/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/foldhash-b8a92f8c10d550f7/lib-foldhash

@@ -1 +0,0 @@
-b7ba5182ce570398

myfsio_core/target/release/.fingerprint/foldhash-b8a92f8c10d550f7/lib-foldhash.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[\"default\", \"std\"]","target":18077926938045032029,"profile":2040997289075261528,"path":9869209539952544870,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\foldhash-b8a92f8c10d550f7\\dep-lib-foldhash","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/generic-array-2462daa120fe5936/build-script-build-script-build

@@ -1 +0,0 @@
-f0a5af4d8a8c7106

myfsio_core/target/release/.fingerprint/generic-array-2462daa120fe5936/build-script-build-script-build.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"more_lengths\"]","declared_features":"[\"more_lengths\", \"serde\", \"zeroize\"]","target":12318548087768197662,"profile":1369601567987815722,"path":13853454403963664247,"deps":[[5398981501050481332,"version_check",false,16419025953046340415]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\generic-array-2462daa120fe5936\\dep-build-script-build-script-build","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/generic-array-2462daa120fe5936/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/generic-array-62216349963f3a3c/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/generic-array-62216349963f3a3c/lib-generic_array

@@ -1 +0,0 @@
-5f316276809d5ca0

myfsio_core/target/release/.fingerprint/generic-array-62216349963f3a3c/lib-generic_array.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"more_lengths\"]","declared_features":"[\"more_lengths\", \"serde\", \"zeroize\"]","target":13084005262763373425,"profile":2040997289075261528,"path":12463275850883329568,"deps":[[857979250431893282,"typenum",false,7416411392359930020],[10520923840501062997,"build_script_build",false,16977603856295925732]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\generic-array-62216349963f3a3c\\dep-lib-generic_array","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/generic-array-a1f36f10f654fafd/run-build-script-build-script-build

@@ -1 +0,0 @@
-e417d28fc1909ceb

myfsio_core/target/release/.fingerprint/generic-array-a1f36f10f654fafd/run-build-script-build-script-build.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"","declared_features":"","target":0,"profile":0,"path":0,"deps":[[10520923840501062997,"build_script_build",false,464306762232604144]],"local":[{"Precalculated":"0.14.7"}],"rustflags":[],"config":0,"compile_kind":0}

myfsio_core/target/release/.fingerprint/hashbrown-510d641b592c306b/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/hashbrown-510d641b592c306b/lib-hashbrown

@@ -1 +0,0 @@
-aec88a641c5288e3

myfsio_core/target/release/.fingerprint/hashbrown-510d641b592c306b/lib-hashbrown.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"allocator-api2\", \"default\", \"default-hasher\", \"equivalent\", \"inline-more\", \"raw-entry\"]","declared_features":"[\"alloc\", \"allocator-api2\", \"core\", \"default\", \"default-hasher\", \"equivalent\", \"inline-more\", \"nightly\", \"raw-entry\", \"rayon\", \"rustc-dep-of-std\", \"rustc-internal-api\", \"serde\"]","target":13796197676120832388,"profile":2040997289075261528,"path":12448322139402656924,"deps":[[5230392855116717286,"equivalent",false,6042941999404782907],[9150530836556604396,"allocator_api2",false,16398368410642502979],[10842263908529601448,"foldhash",false,10953695263156452023]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\hashbrown-510d641b592c306b\\dep-lib-hashbrown","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/heck-b47c94fd2a7e00cb/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/heck-b47c94fd2a7e00cb/lib-heck

@@ -1 +0,0 @@
-ddc0b590ff80762b

myfsio_core/target/release/.fingerprint/heck-b47c94fd2a7e00cb/lib-heck.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":17886154901722686619,"profile":1369601567987815722,"path":8608102977929876445,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\heck-b47c94fd2a7e00cb\\dep-lib-heck","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/hex-253414d2260adcdf/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/hex-253414d2260adcdf/lib-hex

@@ -1 +0,0 @@
-41890ebff4143fa5

myfsio_core/target/release/.fingerprint/hex-253414d2260adcdf/lib-hex.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[\"alloc\", \"default\", \"std\"]","declared_features":"[\"alloc\", \"default\", \"serde\", \"std\"]","target":4242469766639956503,"profile":2040997289075261528,"path":6793865871540733919,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\hex-253414d2260adcdf\\dep-lib-hex","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/hmac-3297e61b9effb758/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/hmac-3297e61b9effb758/lib-hmac

@@ -1 +0,0 @@
-3f45b8d062d94ba4

myfsio_core/target/release/.fingerprint/hmac-3297e61b9effb758/lib-hmac.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[\"reset\", \"std\"]","target":12991177224612424488,"profile":2040997289075261528,"path":17893893568771568113,"deps":[[17475753849556516473,"digest",false,15621022965039188625]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\hmac-3297e61b9effb758\\dep-lib-hmac","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/indoc-0c686c3f403a2566/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/indoc-0c686c3f403a2566/lib-indoc

@@ -1 +0,0 @@
-9896adc8892b3fe4

myfsio_core/target/release/.fingerprint/indoc-0c686c3f403a2566/lib-indoc.json

@@ -1 +0,0 @@
-{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":8726396592336845528,"profile":1369601567987815722,"path":18304219166357541938,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\indoc-0c686c3f403a2566\\dep-lib-indoc","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

myfsio_core/target/release/.fingerprint/libc-5f9f280eeaad3bb3/invoked.timestamp

@@ -1 +0,0 @@
-This file has an mtime of when this was started.

myfsio_core/target/release/.fingerprint/libc-5f9f280eeaad3bb3/lib-libc

@@ -1 +0,0 @@
-3c0b7985f088ae56