Separate Python and Rust into python/ and rust/ with per-stack Dockerfiles

python/myfsio_core/src/crypto.rs

@@ -0,0 +1,192 @@
+use aes_gcm::aead::Aead;
+use aes_gcm::{Aes256Gcm, KeyInit, Nonce};
+use hkdf::Hkdf;
+use pyo3::exceptions::{PyIOError, PyValueError};
+use pyo3::prelude::*;
+use sha2::Sha256;
+use std::fs::File;
+use std::io::{Read, Seek, SeekFrom, Write};
+
+const DEFAULT_CHUNK_SIZE: usize = 65536;
+const HEADER_SIZE: usize = 4;
+
+fn read_exact_chunk(reader: &mut impl Read, buf: &mut [u8]) -> std::io::Result<usize> {
+    let mut filled = 0;
+    while filled < buf.len() {
+        match reader.read(&mut buf[filled..]) {
+            Ok(0) => break,
+            Ok(n) => filled += n,
+            Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
+            Err(e) => return Err(e),
+        }
+    }
+    Ok(filled)
+}
+
+fn derive_chunk_nonce(base_nonce: &[u8], chunk_index: u32) -> Result<[u8; 12], String> {
+    let hkdf = Hkdf::<Sha256>::new(Some(base_nonce), b"chunk_nonce");
+    let mut okm = [0u8; 12];
+    hkdf.expand(&chunk_index.to_be_bytes(), &mut okm)
+        .map_err(|e| format!("HKDF expand failed: {}", e))?;
+    Ok(okm)
+}
+
+#[pyfunction]
+#[pyo3(signature = (input_path, output_path, key, base_nonce, chunk_size=DEFAULT_CHUNK_SIZE))]
+pub fn encrypt_stream_chunked(
+    py: Python<'_>,
+    input_path: &str,
+    output_path: &str,
+    key: &[u8],
+    base_nonce: &[u8],
+    chunk_size: usize,
+) -> PyResult<u32> {
+    if key.len() != 32 {
+        return Err(PyValueError::new_err(format!(
+            "Key must be 32 bytes, got {}",
+            key.len()
+        )));
+    }
+    if base_nonce.len() != 12 {
+        return Err(PyValueError::new_err(format!(
+            "Base nonce must be 12 bytes, got {}",
+            base_nonce.len()
+        )));
+    }
+
+    let chunk_size = if chunk_size == 0 {
+        DEFAULT_CHUNK_SIZE
+    } else {
+        chunk_size
+    };
+
+    let inp = input_path.to_owned();
+    let out = output_path.to_owned();
+    let key_arr: [u8; 32] = key.try_into().unwrap();
+    let nonce_arr: [u8; 12] = base_nonce.try_into().unwrap();
+
+    py.detach(move || {
+        let cipher = Aes256Gcm::new(&key_arr.into());
+
+        let mut infile = File::open(&inp)
+            .map_err(|e| PyIOError::new_err(format!("Failed to open input: {}", e)))?;
+        let mut outfile = File::create(&out)
+            .map_err(|e| PyIOError::new_err(format!("Failed to create output: {}", e)))?;
+
+        outfile
+            .write_all(&[0u8; 4])
+            .map_err(|e| PyIOError::new_err(format!("Failed to write header: {}", e)))?;
+
+        let mut buf = vec![0u8; chunk_size];
+        let mut chunk_index: u32 = 0;
+
+        loop {
+            let n = read_exact_chunk(&mut infile, &mut buf)
+                .map_err(|e| PyIOError::new_err(format!("Failed to read: {}", e)))?;
+            if n == 0 {
+                break;
+            }
+
+            let nonce_bytes = derive_chunk_nonce(&nonce_arr, chunk_index)
+                .map_err(|e| PyValueError::new_err(e))?;
+            let nonce = Nonce::from_slice(&nonce_bytes);
+
+            let encrypted = cipher
+                .encrypt(nonce, &buf[..n])
+                .map_err(|e| PyValueError::new_err(format!("Encrypt failed: {}", e)))?;
+
+            let size = encrypted.len() as u32;
+            outfile
+                .write_all(&size.to_be_bytes())
+                .map_err(|e| PyIOError::new_err(format!("Failed to write chunk size: {}", e)))?;
+            outfile
+                .write_all(&encrypted)
+                .map_err(|e| PyIOError::new_err(format!("Failed to write chunk: {}", e)))?;
+
+            chunk_index += 1;
+        }
+
+        outfile
+            .seek(SeekFrom::Start(0))
+            .map_err(|e| PyIOError::new_err(format!("Failed to seek: {}", e)))?;
+        outfile
+            .write_all(&chunk_index.to_be_bytes())
+            .map_err(|e| PyIOError::new_err(format!("Failed to write chunk count: {}", e)))?;
+
+        Ok(chunk_index)
+    })
+}
+
+#[pyfunction]
+pub fn decrypt_stream_chunked(
+    py: Python<'_>,
+    input_path: &str,
+    output_path: &str,
+    key: &[u8],
+    base_nonce: &[u8],
+) -> PyResult<u32> {
+    if key.len() != 32 {
+        return Err(PyValueError::new_err(format!(
+            "Key must be 32 bytes, got {}",
+            key.len()
+        )));
+    }
+    if base_nonce.len() != 12 {
+        return Err(PyValueError::new_err(format!(
+            "Base nonce must be 12 bytes, got {}",
+            base_nonce.len()
+        )));
+    }
+
+    let inp = input_path.to_owned();
+    let out = output_path.to_owned();
+    let key_arr: [u8; 32] = key.try_into().unwrap();
+    let nonce_arr: [u8; 12] = base_nonce.try_into().unwrap();
+
+    py.detach(move || {
+        let cipher = Aes256Gcm::new(&key_arr.into());
+
+        let mut infile = File::open(&inp)
+            .map_err(|e| PyIOError::new_err(format!("Failed to open input: {}", e)))?;
+        let mut outfile = File::create(&out)
+            .map_err(|e| PyIOError::new_err(format!("Failed to create output: {}", e)))?;
+
+        let mut header = [0u8; HEADER_SIZE];
+        infile
+            .read_exact(&mut header)
+            .map_err(|e| PyIOError::new_err(format!("Failed to read header: {}", e)))?;
+        let chunk_count = u32::from_be_bytes(header);
+
+        let mut size_buf = [0u8; HEADER_SIZE];
+        for chunk_index in 0..chunk_count {
+            infile
+                .read_exact(&mut size_buf)
+                .map_err(|e| {
+                    PyIOError::new_err(format!(
+                        "Failed to read chunk {} size: {}",
+                        chunk_index, e
+                    ))
+                })?;
+            let chunk_size = u32::from_be_bytes(size_buf) as usize;
+
+            let mut encrypted = vec![0u8; chunk_size];
+            infile.read_exact(&mut encrypted).map_err(|e| {
+                PyIOError::new_err(format!("Failed to read chunk {}: {}", chunk_index, e))
+            })?;
+
+            let nonce_bytes = derive_chunk_nonce(&nonce_arr, chunk_index)
+                .map_err(|e| PyValueError::new_err(e))?;
+            let nonce = Nonce::from_slice(&nonce_bytes);
+
+            let decrypted = cipher.decrypt(nonce, encrypted.as_ref()).map_err(|e| {
+                PyValueError::new_err(format!("Decrypt chunk {} failed: {}", chunk_index, e))
+            })?;
+
+            outfile.write_all(&decrypted).map_err(|e| {
+                PyIOError::new_err(format!("Failed to write chunk {}: {}", chunk_index, e))
+            })?;
+        }
+
+        Ok(chunk_count)
+    })
+}

python/myfsio_core/src/hashing.rs

@@ -0,0 +1,90 @@
+use md5::{Digest, Md5};
+use pyo3::exceptions::PyIOError;
+use pyo3::prelude::*;
+use sha2::Sha256;
+use std::fs::File;
+use std::io::Read;
+
+const CHUNK_SIZE: usize = 65536;
+
+#[pyfunction]
+pub fn md5_file(py: Python<'_>, path: &str) -> PyResult<String> {
+    let path = path.to_owned();
+    py.detach(move || {
+        let mut file = File::open(&path)
+            .map_err(|e| PyIOError::new_err(format!("Failed to open file: {}", e)))?;
+        let mut hasher = Md5::new();
+        let mut buf = vec![0u8; CHUNK_SIZE];
+        loop {
+            let n = file
+                .read(&mut buf)
+                .map_err(|e| PyIOError::new_err(format!("Failed to read file: {}", e)))?;
+            if n == 0 {
+                break;
+            }
+            hasher.update(&buf[..n]);
+        }
+        Ok(format!("{:x}", hasher.finalize()))
+    })
+}
+
+#[pyfunction]
+pub fn md5_bytes(data: &[u8]) -> String {
+    let mut hasher = Md5::new();
+    hasher.update(data);
+    format!("{:x}", hasher.finalize())
+}
+
+#[pyfunction]
+pub fn sha256_file(py: Python<'_>, path: &str) -> PyResult<String> {
+    let path = path.to_owned();
+    py.detach(move || {
+        let mut file = File::open(&path)
+            .map_err(|e| PyIOError::new_err(format!("Failed to open file: {}", e)))?;
+        let mut hasher = Sha256::new();
+        let mut buf = vec![0u8; CHUNK_SIZE];
+        loop {
+            let n = file
+                .read(&mut buf)
+                .map_err(|e| PyIOError::new_err(format!("Failed to read file: {}", e)))?;
+            if n == 0 {
+                break;
+            }
+            hasher.update(&buf[..n]);
+        }
+        Ok(format!("{:x}", hasher.finalize()))
+    })
+}
+
+#[pyfunction]
+pub fn sha256_bytes(data: &[u8]) -> String {
+    let mut hasher = Sha256::new();
+    hasher.update(data);
+    format!("{:x}", hasher.finalize())
+}
+
+#[pyfunction]
+pub fn md5_sha256_file(py: Python<'_>, path: &str) -> PyResult<(String, String)> {
+    let path = path.to_owned();
+    py.detach(move || {
+        let mut file = File::open(&path)
+            .map_err(|e| PyIOError::new_err(format!("Failed to open file: {}", e)))?;
+        let mut md5_hasher = Md5::new();
+        let mut sha_hasher = Sha256::new();
+        let mut buf = vec![0u8; CHUNK_SIZE];
+        loop {
+            let n = file
+                .read(&mut buf)
+                .map_err(|e| PyIOError::new_err(format!("Failed to read file: {}", e)))?;
+            if n == 0 {
+                break;
+            }
+            md5_hasher.update(&buf[..n]);
+            sha_hasher.update(&buf[..n]);
+        }
+        Ok((
+            format!("{:x}", md5_hasher.finalize()),
+            format!("{:x}", sha_hasher.finalize()),
+        ))
+    })
+}

python/myfsio_core/src/lib.rs

@@ -0,0 +1,51 @@
+mod crypto;
+mod hashing;
+mod metadata;
+mod sigv4;
+mod storage;
+mod streaming;
+mod validation;
+
+use pyo3::prelude::*;
+
+#[pymodule]
+mod myfsio_core {
+    use super::*;
+
+    #[pymodule_init]
+    fn init(m: &Bound<'_, PyModule>) -> PyResult<()> {
+        m.add_function(wrap_pyfunction!(sigv4::verify_sigv4_signature, m)?)?;
+        m.add_function(wrap_pyfunction!(sigv4::derive_signing_key, m)?)?;
+        m.add_function(wrap_pyfunction!(sigv4::compute_signature, m)?)?;
+        m.add_function(wrap_pyfunction!(sigv4::build_string_to_sign, m)?)?;
+        m.add_function(wrap_pyfunction!(sigv4::constant_time_compare, m)?)?;
+        m.add_function(wrap_pyfunction!(sigv4::clear_signing_key_cache, m)?)?;
+
+        m.add_function(wrap_pyfunction!(hashing::md5_file, m)?)?;
+        m.add_function(wrap_pyfunction!(hashing::md5_bytes, m)?)?;
+        m.add_function(wrap_pyfunction!(hashing::sha256_file, m)?)?;
+        m.add_function(wrap_pyfunction!(hashing::sha256_bytes, m)?)?;
+        m.add_function(wrap_pyfunction!(hashing::md5_sha256_file, m)?)?;
+
+        m.add_function(wrap_pyfunction!(validation::validate_object_key, m)?)?;
+        m.add_function(wrap_pyfunction!(validation::validate_bucket_name, m)?)?;
+
+        m.add_function(wrap_pyfunction!(metadata::read_index_entry, m)?)?;
+
+        m.add_function(wrap_pyfunction!(storage::write_index_entry, m)?)?;
+        m.add_function(wrap_pyfunction!(storage::delete_index_entry, m)?)?;
+        m.add_function(wrap_pyfunction!(storage::check_bucket_contents, m)?)?;
+        m.add_function(wrap_pyfunction!(storage::shallow_scan, m)?)?;
+        m.add_function(wrap_pyfunction!(storage::bucket_stats_scan, m)?)?;
+        m.add_function(wrap_pyfunction!(storage::search_objects_scan, m)?)?;
+        m.add_function(wrap_pyfunction!(storage::build_object_cache, m)?)?;
+
+        m.add_function(wrap_pyfunction!(streaming::stream_to_file_with_md5, m)?)?;
+        m.add_function(wrap_pyfunction!(streaming::assemble_parts_with_md5, m)?)?;
+
+        m.add_function(wrap_pyfunction!(crypto::encrypt_stream_chunked, m)?)?;
+        m.add_function(wrap_pyfunction!(crypto::decrypt_stream_chunked, m)?)?;
+
+        Ok(())
+    }
+}

python/myfsio_core/src/metadata.rs

@@ -0,0 +1,71 @@
+use pyo3::exceptions::PyValueError;
+use pyo3::prelude::*;
+use pyo3::types::{PyDict, PyList, PyString};
+use serde_json::Value;
+use std::fs;
+
+const MAX_DEPTH: u32 = 64;
+
+fn value_to_py(py: Python<'_>, v: &Value, depth: u32) -> PyResult<Py<PyAny>> {
+    if depth > MAX_DEPTH {
+        return Err(PyValueError::new_err("JSON nesting too deep"));
+    }
+    match v {
+        Value::Null => Ok(py.None()),
+        Value::Bool(b) => Ok((*b).into_pyobject(py)?.to_owned().into_any().unbind()),
+        Value::Number(n) => {
+            if let Some(i) = n.as_i64() {
+                Ok(i.into_pyobject(py)?.into_any().unbind())
+            } else if let Some(f) = n.as_f64() {
+                Ok(f.into_pyobject(py)?.into_any().unbind())
+            } else {
+                Ok(py.None())
+            }
+        }
+        Value::String(s) => Ok(PyString::new(py, s).into_any().unbind()),
+        Value::Array(arr) => {
+            let list = PyList::empty(py);
+            for item in arr {
+                list.append(value_to_py(py, item, depth + 1)?)?;
+            }
+            Ok(list.into_any().unbind())
+        }
+        Value::Object(map) => {
+            let dict = PyDict::new(py);
+            for (k, val) in map {
+                dict.set_item(k, value_to_py(py, val, depth + 1)?)?;
+            }
+            Ok(dict.into_any().unbind())
+        }
+    }
+}
+
+#[pyfunction]
+pub fn read_index_entry(
+    py: Python<'_>,
+    path: &str,
+    entry_name: &str,
+) -> PyResult<Option<Py<PyAny>>> {
+    let path_owned = path.to_owned();
+    let entry_owned = entry_name.to_owned();
+
+    let entry: Option<Value> = py.detach(move || -> PyResult<Option<Value>> {
+        let content = match fs::read_to_string(&path_owned) {
+            Ok(c) => c,
+            Err(_) => return Ok(None),
+        };
+        let parsed: Value = match serde_json::from_str(&content) {
+            Ok(v) => v,
+            Err(_) => return Ok(None),
+        };
+        match parsed {
+            Value::Object(mut map) => Ok(map.remove(&entry_owned)),
+            _ => Ok(None),
+        }
+    })?;
+
+    match entry {
+        Some(val) => Ok(Some(value_to_py(py, &val, 0)?)),
+        None => Ok(None),
+    }
+}

python/myfsio_core/src/sigv4.rs

@@ -0,0 +1,193 @@
+use hmac::{Hmac, Mac};
+use lru::LruCache;
+use parking_lot::Mutex;
+use percent_encoding::{percent_encode, AsciiSet, NON_ALPHANUMERIC};
+use pyo3::prelude::*;
+use sha2::{Digest, Sha256};
+use std::num::NonZeroUsize;
+use std::sync::LazyLock;
+use std::time::Instant;
+
+type HmacSha256 = Hmac<Sha256>;
+
+struct CacheEntry {
+    key: Vec<u8>,
+    created: Instant,
+}
+
+static SIGNING_KEY_CACHE: LazyLock<Mutex<LruCache<(String, String, String, String), CacheEntry>>> =
+    LazyLock::new(|| Mutex::new(LruCache::new(NonZeroUsize::new(256).unwrap())));
+
+const CACHE_TTL_SECS: u64 = 60;
+
+const AWS_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC
+    .remove(b'-')
+    .remove(b'_')
+    .remove(b'.')
+    .remove(b'~');
+
+fn hmac_sha256(key: &[u8], msg: &[u8]) -> Vec<u8> {
+    let mut mac = HmacSha256::new_from_slice(key).expect("HMAC key length is always valid");
+    mac.update(msg);
+    mac.finalize().into_bytes().to_vec()
+}
+
+fn sha256_hex(data: &[u8]) -> String {
+    let mut hasher = Sha256::new();
+    hasher.update(data);
+    hex::encode(hasher.finalize())
+}
+
+fn aws_uri_encode(input: &str) -> String {
+    percent_encode(input.as_bytes(), AWS_ENCODE_SET).to_string()
+}
+
+fn derive_signing_key_cached(
+    secret_key: &str,
+    date_stamp: &str,
+    region: &str,
+    service: &str,
+) -> Vec<u8> {
+    let cache_key = (
+        secret_key.to_owned(),
+        date_stamp.to_owned(),
+        region.to_owned(),
+        service.to_owned(),
+    );
+
+    {
+        let mut cache = SIGNING_KEY_CACHE.lock();
+        if let Some(entry) = cache.get(&cache_key) {
+            if entry.created.elapsed().as_secs() < CACHE_TTL_SECS {
+                return entry.key.clone();
+            }
+            cache.pop(&cache_key);
+        }
+    }
+
+    let k_date = hmac_sha256(format!("AWS4{}", secret_key).as_bytes(), date_stamp.as_bytes());
+    let k_region = hmac_sha256(&k_date, region.as_bytes());
+    let k_service = hmac_sha256(&k_region, service.as_bytes());
+    let k_signing = hmac_sha256(&k_service, b"aws4_request");
+
+    {
+        let mut cache = SIGNING_KEY_CACHE.lock();
+        cache.put(
+            cache_key,
+            CacheEntry {
+                key: k_signing.clone(),
+                created: Instant::now(),
+            },
+        );
+    }
+
+    k_signing
+}
+
+fn constant_time_compare_inner(a: &[u8], b: &[u8]) -> bool {
+    if a.len() != b.len() {
+        return false;
+    }
+    let mut result: u8 = 0;
+    for (x, y) in a.iter().zip(b.iter()) {
+        result |= x ^ y;
+    }
+    result == 0
+}
+
+#[pyfunction]
+pub fn verify_sigv4_signature(
+    method: &str,
+    canonical_uri: &str,
+    query_params: Vec<(String, String)>,
+    signed_headers_str: &str,
+    header_values: Vec<(String, String)>,
+    payload_hash: &str,
+    amz_date: &str,
+    date_stamp: &str,
+    region: &str,
+    service: &str,
+    secret_key: &str,
+    provided_signature: &str,
+) -> bool {
+    let mut sorted_params = query_params;
+    sorted_params.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)));
+
+    let canonical_query_string = sorted_params
+        .iter()
+        .map(|(k, v)| format!("{}={}", aws_uri_encode(k), aws_uri_encode(v)))
+        .collect::<Vec<_>>()
+        .join("&");
+
+    let mut canonical_headers = String::new();
+    for (name, value) in &header_values {
+        let lower_name = name.to_lowercase();
+        let normalized = value.split_whitespace().collect::<Vec<_>>().join(" ");
+        let final_value = if lower_name == "expect" && normalized.is_empty() {
+            "100-continue"
+        } else {
+            &normalized
+        };
+        canonical_headers.push_str(&lower_name);
+        canonical_headers.push(':');
+        canonical_headers.push_str(final_value);
+        canonical_headers.push('\n');
+    }
+
+    let canonical_request = format!(
+        "{}\n{}\n{}\n{}\n{}\n{}",
+        method, canonical_uri, canonical_query_string, canonical_headers, signed_headers_str, payload_hash
+    );
+
+    let credential_scope = format!("{}/{}/{}/aws4_request", date_stamp, region, service);
+    let cr_hash = sha256_hex(canonical_request.as_bytes());
+    let string_to_sign = format!(
+        "AWS4-HMAC-SHA256\n{}\n{}\n{}",
+        amz_date, credential_scope, cr_hash
+    );
+
+    let signing_key = derive_signing_key_cached(secret_key, date_stamp, region, service);
+    let calculated = hmac_sha256(&signing_key, string_to_sign.as_bytes());
+    let calculated_hex = hex::encode(&calculated);
+
+    constant_time_compare_inner(calculated_hex.as_bytes(), provided_signature.as_bytes())
+}
+
+#[pyfunction]
+pub fn derive_signing_key(
+    secret_key: &str,
+    date_stamp: &str,
+    region: &str,
+    service: &str,
+) -> Vec<u8> {
+    derive_signing_key_cached(secret_key, date_stamp, region, service)
+}
+
+#[pyfunction]
+pub fn compute_signature(signing_key: &[u8], string_to_sign: &str) -> String {
+    let sig = hmac_sha256(signing_key, string_to_sign.as_bytes());
+    hex::encode(sig)
+}
+
+#[pyfunction]
+pub fn build_string_to_sign(
+    amz_date: &str,
+    credential_scope: &str,
+    canonical_request: &str,
+) -> String {
+    let cr_hash = sha256_hex(canonical_request.as_bytes());
+    format!(
+        "AWS4-HMAC-SHA256\n{}\n{}\n{}",
+        amz_date, credential_scope, cr_hash
+    )
+}
+
+#[pyfunction]
+pub fn constant_time_compare(a: &str, b: &str) -> bool {
+    constant_time_compare_inner(a.as_bytes(), b.as_bytes())
+}
+
+#[pyfunction]
+pub fn clear_signing_key_cache() {
+    SIGNING_KEY_CACHE.lock().clear();
+}

python/myfsio_core/src/storage.rs

@@ -0,0 +1,817 @@
+use pyo3::exceptions::PyIOError;
+use pyo3::prelude::*;
+use pyo3::types::{PyDict, PyList, PyString, PyTuple};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::fs;
+use std::path::Path;
+use std::time::SystemTime;
+
+const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"];
+
+fn system_time_to_epoch(t: SystemTime) -> f64 {
+    t.duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_secs_f64())
+        .unwrap_or(0.0)
+}
+
+fn extract_etag_from_meta_bytes(content: &[u8]) -> Option<String> {
+    let marker = b"\"__etag__\"";
+    let idx = content.windows(marker.len()).position(|w| w == marker)?;
+    let after = &content[idx + marker.len()..];
+    let start = after.iter().position(|&b| b == b'"')? + 1;
+    let rest = &after[start..];
+    let end = rest.iter().position(|&b| b == b'"')?;
+    std::str::from_utf8(&rest[..end]).ok().map(|s| s.to_owned())
+}
+
+fn has_any_file(root: &str) -> bool {
+    let root_path = Path::new(root);
+    if !root_path.is_dir() {
+        return false;
+    }
+    let mut stack = vec![root_path.to_path_buf()];
+    while let Some(current) = stack.pop() {
+        let entries = match fs::read_dir(&current) {
+            Ok(e) => e,
+            Err(_) => continue,
+        };
+        for entry_result in entries {
+            let entry = match entry_result {
+                Ok(e) => e,
+                Err(_) => continue,
+            };
+            let ft = match entry.file_type() {
+                Ok(ft) => ft,
+                Err(_) => continue,
+            };
+            if ft.is_file() {
+                return true;
+            }
+            if ft.is_dir() && !ft.is_symlink() {
+                stack.push(entry.path());
+            }
+        }
+    }
+    false
+}
+
+#[pyfunction]
+pub fn write_index_entry(
+    py: Python<'_>,
+    path: &str,
+    entry_name: &str,
+    entry_data_json: &str,
+) -> PyResult<()> {
+    let path_owned = path.to_owned();
+    let entry_owned = entry_name.to_owned();
+    let data_owned = entry_data_json.to_owned();
+
+    py.detach(move || -> PyResult<()> {
+        let entry_value: Value = serde_json::from_str(&data_owned)
+            .map_err(|e| PyIOError::new_err(format!("Failed to parse entry data: {}", e)))?;
+
+        if let Some(parent) = Path::new(&path_owned).parent() {
+            let _ = fs::create_dir_all(parent);
+        }
+
+        let mut index_data: serde_json::Map<String, Value> = match fs::read_to_string(&path_owned)
+        {
+            Ok(content) => serde_json::from_str(&content).unwrap_or_default(),
+            Err(_) => serde_json::Map::new(),
+        };
+
+        index_data.insert(entry_owned, entry_value);
+
+        let serialized = serde_json::to_string(&Value::Object(index_data))
+            .map_err(|e| PyIOError::new_err(format!("Failed to serialize index: {}", e)))?;
+
+        fs::write(&path_owned, serialized)
+            .map_err(|e| PyIOError::new_err(format!("Failed to write index: {}", e)))?;
+
+        Ok(())
+    })
+}
+
+#[pyfunction]
+pub fn delete_index_entry(py: Python<'_>, path: &str, entry_name: &str) -> PyResult<bool> {
+    let path_owned = path.to_owned();
+    let entry_owned = entry_name.to_owned();
+
+    py.detach(move || -> PyResult<bool> {
+        let content = match fs::read_to_string(&path_owned) {
+            Ok(c) => c,
+            Err(_) => return Ok(false),
+        };
+
+        let mut index_data: serde_json::Map<String, Value> =
+            match serde_json::from_str(&content) {
+                Ok(v) => v,
+                Err(_) => return Ok(false),
+            };
+
+        if index_data.remove(&entry_owned).is_none() {
+            return Ok(false);
+        }
+
+        if index_data.is_empty() {
+            let _ = fs::remove_file(&path_owned);
+            return Ok(true);
+        }
+
+        let serialized = serde_json::to_string(&Value::Object(index_data))
+            .map_err(|e| PyIOError::new_err(format!("Failed to serialize index: {}", e)))?;
+
+        fs::write(&path_owned, serialized)
+            .map_err(|e| PyIOError::new_err(format!("Failed to write index: {}", e)))?;
+
+        Ok(true)
+    })
+}
+
+#[pyfunction]
+pub fn check_bucket_contents(
+    py: Python<'_>,
+    bucket_path: &str,
+    version_roots: Vec<String>,
+    multipart_roots: Vec<String>,
+) -> PyResult<(bool, bool, bool)> {
+    let bucket_owned = bucket_path.to_owned();
+
+    py.detach(move || -> PyResult<(bool, bool, bool)> {
+        let mut has_objects = false;
+        let bucket_p = Path::new(&bucket_owned);
+        if bucket_p.is_dir() {
+            let mut stack = vec![bucket_p.to_path_buf()];
+            'obj_scan: while let Some(current) = stack.pop() {
+                let is_root = current == bucket_p;
+                let entries = match fs::read_dir(&current) {
+                    Ok(e) => e,
+                    Err(_) => continue,
+                };
+                for entry_result in entries {
+                    let entry = match entry_result {
+                        Ok(e) => e,
+                        Err(_) => continue,
+                    };
+                    let ft = match entry.file_type() {
+                        Ok(ft) => ft,
+                        Err(_) => continue,
+                    };
+                    if is_root {
+                        if let Some(name) = entry.file_name().to_str() {
+                            if INTERNAL_FOLDERS.contains(&name) {
+                                continue;
+                            }
+                        }
+                    }
+                    if ft.is_file() && !ft.is_symlink() {
+                        has_objects = true;
+                        break 'obj_scan;
+                    }
+                    if ft.is_dir() && !ft.is_symlink() {
+                        stack.push(entry.path());
+                    }
+                }
+            }
+        }
+
+        let mut has_versions = false;
+        for root in &version_roots {
+            if has_versions {
+                break;
+            }
+            has_versions = has_any_file(root);
+        }
+
+        let mut has_multipart = false;
+        for root in &multipart_roots {
+            if has_multipart {
+                break;
+            }
+            has_multipart = has_any_file(root);
+        }
+
+        Ok((has_objects, has_versions, has_multipart))
+    })
+}
+
+#[pyfunction]
+pub fn shallow_scan(
+    py: Python<'_>,
+    target_dir: &str,
+    prefix: &str,
+    meta_cache_json: &str,
+) -> PyResult<Py<PyAny>> {
+    let target_owned = target_dir.to_owned();
+    let prefix_owned = prefix.to_owned();
+    let cache_owned = meta_cache_json.to_owned();
+
+    let result: (
+        Vec<(String, u64, f64, Option<String>)>,
+        Vec<String>,
+        Vec<(String, bool)>,
+    ) = py.detach(move || -> PyResult<(
+        Vec<(String, u64, f64, Option<String>)>,
+        Vec<String>,
+        Vec<(String, bool)>,
+    )> {
+        let meta_cache: HashMap<String, String> =
+            serde_json::from_str(&cache_owned).unwrap_or_default();
+
+        let mut files: Vec<(String, u64, f64, Option<String>)> = Vec::new();
+        let mut dirs: Vec<String> = Vec::new();
+
+        let entries = match fs::read_dir(&target_owned) {
+            Ok(e) => e,
+            Err(_) => return Ok((files, dirs, Vec::new())),
+        };
+
+        for entry_result in entries {
+            let entry = match entry_result {
+                Ok(e) => e,
+                Err(_) => continue,
+            };
+            let name = match entry.file_name().into_string() {
+                Ok(n) => n,
+                Err(_) => continue,
+            };
+            if INTERNAL_FOLDERS.contains(&name.as_str()) {
+                continue;
+            }
+            let ft = match entry.file_type() {
+                Ok(ft) => ft,
+                Err(_) => continue,
+            };
+            if ft.is_dir() && !ft.is_symlink() {
+                let cp = format!("{}{}/", prefix_owned, name);
+                dirs.push(cp);
+            } else if ft.is_file() && !ft.is_symlink() {
+                let key = format!("{}{}", prefix_owned, name);
+                let md = match entry.metadata() {
+                    Ok(m) => m,
+                    Err(_) => continue,
+                };
+                let size = md.len();
+                let mtime = md
+                    .modified()
+                    .map(system_time_to_epoch)
+                    .unwrap_or(0.0);
+                let etag = meta_cache.get(&key).cloned();
+                files.push((key, size, mtime, etag));
+            }
+        }
+
+        files.sort_by(|a, b| a.0.cmp(&b.0));
+        dirs.sort();
+
+        let mut merged: Vec<(String, bool)> = Vec::with_capacity(files.len() + dirs.len());
+        let mut fi = 0;
+        let mut di = 0;
+        while fi < files.len() && di < dirs.len() {
+            if files[fi].0 <= dirs[di] {
+                merged.push((files[fi].0.clone(), false));
+                fi += 1;
+            } else {
+                merged.push((dirs[di].clone(), true));
+                di += 1;
+            }
+        }
+        while fi < files.len() {
+            merged.push((files[fi].0.clone(), false));
+            fi += 1;
+        }
+        while di < dirs.len() {
+            merged.push((dirs[di].clone(), true));
+            di += 1;
+        }
+
+        Ok((files, dirs, merged))
+    })?;
+
+    let (files, dirs, merged) = result;
+
+    let dict = PyDict::new(py);
+
+    let files_list = PyList::empty(py);
+    for (key, size, mtime, etag) in &files {
+        let etag_py: Py<PyAny> = match etag {
+            Some(e) => PyString::new(py, e).into_any().unbind(),
+            None => py.None(),
+        };
+        let tuple = PyTuple::new(py, &[
+            PyString::new(py, key).into_any().unbind(),
+            size.into_pyobject(py)?.into_any().unbind(),
+            mtime.into_pyobject(py)?.into_any().unbind(),
+            etag_py,
+        ])?;
+        files_list.append(tuple)?;
+    }
+    dict.set_item("files", files_list)?;
+
+    let dirs_list = PyList::empty(py);
+    for d in &dirs {
+        dirs_list.append(PyString::new(py, d))?;
+    }
+    dict.set_item("dirs", dirs_list)?;
+
+    let merged_list = PyList::empty(py);
+    for (key, is_dir) in &merged {
+        let bool_obj: Py<PyAny> = if *is_dir {
+            true.into_pyobject(py)?.to_owned().into_any().unbind()
+        } else {
+            false.into_pyobject(py)?.to_owned().into_any().unbind()
+        };
+        let tuple = PyTuple::new(py, &[
+            PyString::new(py, key).into_any().unbind(),
+            bool_obj,
+        ])?;
+        merged_list.append(tuple)?;
+    }
+    dict.set_item("merged_keys", merged_list)?;
+
+    Ok(dict.into_any().unbind())
+}
+
+#[pyfunction]
+pub fn bucket_stats_scan(
+    py: Python<'_>,
+    bucket_path: &str,
+    versions_root: &str,
+) -> PyResult<(u64, u64, u64, u64)> {
+    let bucket_owned = bucket_path.to_owned();
+    let versions_owned = versions_root.to_owned();
+
+    py.detach(move || -> PyResult<(u64, u64, u64, u64)> {
+        let mut object_count: u64 = 0;
+        let mut total_bytes: u64 = 0;
+
+        let bucket_p = Path::new(&bucket_owned);
+        if bucket_p.is_dir() {
+            let mut stack = vec![bucket_p.to_path_buf()];
+            while let Some(current) = stack.pop() {
+                let is_root = current == bucket_p;
+                let entries = match fs::read_dir(&current) {
+                    Ok(e) => e,
+                    Err(_) => continue,
+                };
+                for entry_result in entries {
+                    let entry = match entry_result {
+                        Ok(e) => e,
+                        Err(_) => continue,
+                    };
+                    if is_root {
+                        if let Some(name) = entry.file_name().to_str() {
+                            if INTERNAL_FOLDERS.contains(&name) {
+                                continue;
+                            }
+                        }
+                    }
+                    let ft = match entry.file_type() {
+                        Ok(ft) => ft,
+                        Err(_) => continue,
+                    };
+                    if ft.is_dir() && !ft.is_symlink() {
+                        stack.push(entry.path());
+                    } else if ft.is_file() && !ft.is_symlink() {
+                        object_count += 1;
+                        if let Ok(md) = entry.metadata() {
+                            total_bytes += md.len();
+                        }
+                    }
+                }
+            }
+        }
+
+        let mut version_count: u64 = 0;
+        let mut version_bytes: u64 = 0;
+
+        let versions_p = Path::new(&versions_owned);
+        if versions_p.is_dir() {
+            let mut stack = vec![versions_p.to_path_buf()];
+            while let Some(current) = stack.pop() {
+                let entries = match fs::read_dir(&current) {
+                    Ok(e) => e,
+                    Err(_) => continue,
+                };
+                for entry_result in entries {
+                    let entry = match entry_result {
+                        Ok(e) => e,
+                        Err(_) => continue,
+                    };
+                    let ft = match entry.file_type() {
+                        Ok(ft) => ft,
+                        Err(_) => continue,
+                    };
+                    if ft.is_dir() && !ft.is_symlink() {
+                        stack.push(entry.path());
+                    } else if ft.is_file() && !ft.is_symlink() {
+                        if let Some(name) = entry.file_name().to_str() {
+                            if name.ends_with(".bin") {
+                                version_count += 1;
+                                if let Ok(md) = entry.metadata() {
+                                    version_bytes += md.len();
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        Ok((object_count, total_bytes, version_count, version_bytes))
+    })
+}
+
+#[pyfunction]
+#[pyo3(signature = (bucket_path, search_root, query, limit))]
+pub fn search_objects_scan(
+    py: Python<'_>,
+    bucket_path: &str,
+    search_root: &str,
+    query: &str,
+    limit: usize,
+) -> PyResult<Py<PyAny>> {
+    let bucket_owned = bucket_path.to_owned();
+    let search_owned = search_root.to_owned();
+    let query_owned = query.to_owned();
+
+    let result: (Vec<(String, u64, f64)>, bool) = py.detach(
+        move || -> PyResult<(Vec<(String, u64, f64)>, bool)> {
+            let query_lower = query_owned.to_lowercase();
+            let bucket_len = bucket_owned.len() + 1;
+            let scan_limit = limit * 4;
+            let mut matched: usize = 0;
+            let mut results: Vec<(String, u64, f64)> = Vec::new();
+
+            let search_p = Path::new(&search_owned);
+            if !search_p.is_dir() {
+                return Ok((results, false));
+            }
+
+            let bucket_p = Path::new(&bucket_owned);
+            let mut stack = vec![search_p.to_path_buf()];
+
+            'scan: while let Some(current) = stack.pop() {
+                let is_bucket_root = current == bucket_p;
+                let entries = match fs::read_dir(&current) {
+                    Ok(e) => e,
+                    Err(_) => continue,
+                };
+                for entry_result in entries {
+                    let entry = match entry_result {
+                        Ok(e) => e,
+                        Err(_) => continue,
+                    };
+                    if is_bucket_root {
+                        if let Some(name) = entry.file_name().to_str() {
+                            if INTERNAL_FOLDERS.contains(&name) {
+                                continue;
+                            }
+                        }
+                    }
+                    let ft = match entry.file_type() {
+                        Ok(ft) => ft,
+                        Err(_) => continue,
+                    };
+                    if ft.is_dir() && !ft.is_symlink() {
+                        stack.push(entry.path());
+                    } else if ft.is_file() && !ft.is_symlink() {
+                        let full_path = entry.path();
+                        let full_str = full_path.to_string_lossy();
+                        if full_str.len() <= bucket_len {
+                            continue;
+                        }
+                        let key = full_str[bucket_len..].replace('\\', "/");
+                        if key.to_lowercase().contains(&query_lower) {
+                            if let Ok(md) = entry.metadata() {
+                                let size = md.len();
+                                let mtime = md
+                                    .modified()
+                                    .map(system_time_to_epoch)
+                                    .unwrap_or(0.0);
+                                results.push((key, size, mtime));
+                                matched += 1;
+                            }
+                        }
+                        if matched >= scan_limit {
+                            break 'scan;
+                        }
+                    }
+                }
+            }
+
+            results.sort_by(|a, b| a.0.cmp(&b.0));
+            let truncated = results.len() > limit;
+            results.truncate(limit);
+
+            Ok((results, truncated))
+        },
+    )?;
+
+    let (results, truncated) = result;
+
+    let dict = PyDict::new(py);
+
+    let results_list = PyList::empty(py);
+    for (key, size, mtime) in &results {
+        let tuple = PyTuple::new(py, &[
+            PyString::new(py, key).into_any().unbind(),
+            size.into_pyobject(py)?.into_any().unbind(),
+            mtime.into_pyobject(py)?.into_any().unbind(),
+        ])?;
+        results_list.append(tuple)?;
+    }
+    dict.set_item("results", results_list)?;
+    dict.set_item("truncated", truncated)?;
+
+    Ok(dict.into_any().unbind())
+}
+
+#[pyfunction]
+pub fn build_object_cache(
+    py: Python<'_>,
+    bucket_path: &str,
+    meta_root: &str,
+    etag_index_path: &str,
+) -> PyResult<Py<PyAny>> {
+    let bucket_owned = bucket_path.to_owned();
+    let meta_owned = meta_root.to_owned();
+    let index_path_owned = etag_index_path.to_owned();
+
+    let result: (HashMap<String, String>, Vec<(String, u64, f64, Option<String>)>, bool) =
+        py.detach(move || -> PyResult<(
+            HashMap<String, String>,
+            Vec<(String, u64, f64, Option<String>)>,
+            bool,
+        )> {
+            let mut meta_cache: HashMap<String, String> = HashMap::new();
+            let mut index_mtime: f64 = 0.0;
+            let mut etag_cache_changed = false;
+
+            let index_p = Path::new(&index_path_owned);
+            if index_p.is_file() {
+                if let Ok(md) = fs::metadata(&index_path_owned) {
+                    index_mtime = md
+                        .modified()
+                        .map(system_time_to_epoch)
+                        .unwrap_or(0.0);
+                }
+                if let Ok(content) = fs::read_to_string(&index_path_owned) {
+                    if let Ok(parsed) = serde_json::from_str::<HashMap<String, String>>(&content) {
+                        meta_cache = parsed;
+                    }
+                }
+            }
+
+            let meta_p = Path::new(&meta_owned);
+            let mut needs_rebuild = false;
+
+            if meta_p.is_dir() && index_mtime > 0.0 {
+                fn check_newer(dir: &Path, index_mtime: f64) -> bool {
+                    let entries = match fs::read_dir(dir) {
+                        Ok(e) => e,
+                        Err(_) => return false,
+                    };
+                    for entry_result in entries {
+                        let entry = match entry_result {
+                            Ok(e) => e,
+                            Err(_) => continue,
+                        };
+                        let ft = match entry.file_type() {
+                            Ok(ft) => ft,
+                            Err(_) => continue,
+                        };
+                        if ft.is_dir() && !ft.is_symlink() {
+                            if check_newer(&entry.path(), index_mtime) {
+                                return true;
+                            }
+                        } else if ft.is_file() {
+                            if let Some(name) = entry.file_name().to_str() {
+                                if name.ends_with(".meta.json") || name == "_index.json" {
+                                    if let Ok(md) = entry.metadata() {
+                                        let mt = md
+                                            .modified()
+                                            .map(system_time_to_epoch)
+                                            .unwrap_or(0.0);
+                                        if mt > index_mtime {
+                                            return true;
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    false
+                }
+                needs_rebuild = check_newer(meta_p, index_mtime);
+            } else if meta_cache.is_empty() {
+                needs_rebuild = true;
+            }
+
+            if needs_rebuild && meta_p.is_dir() {
+                let meta_str = meta_owned.clone();
+                let meta_len = meta_str.len() + 1;
+                let mut index_files: Vec<String> = Vec::new();
+                let mut legacy_meta_files: Vec<(String, String)> = Vec::new();
+
+                fn collect_meta(
+                    dir: &Path,
+                    meta_len: usize,
+                    index_files: &mut Vec<String>,
+                    legacy_meta_files: &mut Vec<(String, String)>,
+                ) {
+                    let entries = match fs::read_dir(dir) {
+                        Ok(e) => e,
+                        Err(_) => return,
+                    };
+                    for entry_result in entries {
+                        let entry = match entry_result {
+                            Ok(e) => e,
+                            Err(_) => continue,
+                        };
+                        let ft = match entry.file_type() {
+                            Ok(ft) => ft,
+                            Err(_) => continue,
+                        };
+                        if ft.is_dir() && !ft.is_symlink() {
+                            collect_meta(&entry.path(), meta_len, index_files, legacy_meta_files);
+                        } else if ft.is_file() {
+                            if let Some(name) = entry.file_name().to_str() {
+                                let full = entry.path().to_string_lossy().to_string();
+                                if name == "_index.json" {
+                                    index_files.push(full);
+                                } else if name.ends_with(".meta.json") {
+                                    if full.len() > meta_len {
+                                        let rel = &full[meta_len..];
+                                        let key = if rel.len() > 10 {
+                                            rel[..rel.len() - 10].replace('\\', "/")
+                                        } else {
+                                            continue;
+                                        };
+                                        legacy_meta_files.push((key, full));
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+
+                collect_meta(
+                    meta_p,
+                    meta_len,
+                    &mut index_files,
+                    &mut legacy_meta_files,
+                );
+
+                meta_cache.clear();
+
+                for idx_path in &index_files {
+                    if let Ok(content) = fs::read_to_string(idx_path) {
+                        if let Ok(idx_data) = serde_json::from_str::<HashMap<String, Value>>(&content) {
+                            let rel_dir = if idx_path.len() > meta_len {
+                                let r = &idx_path[meta_len..];
+                                r.replace('\\', "/")
+                            } else {
+                                String::new()
+                            };
+                            let dir_prefix = if rel_dir.ends_with("/_index.json") {
+                                &rel_dir[..rel_dir.len() - "/_index.json".len()]
+                            } else {
+                                ""
+                            };
+                            for (entry_name, entry_data) in &idx_data {
+                                let key = if dir_prefix.is_empty() {
+                                    entry_name.clone()
+                                } else {
+                                    format!("{}/{}", dir_prefix, entry_name)
+                                };
+                                if let Some(meta_obj) = entry_data.get("metadata") {
+                                    if let Some(etag) = meta_obj.get("__etag__") {
+                                        if let Some(etag_str) = etag.as_str() {
+                                            meta_cache.insert(key, etag_str.to_owned());
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+
+                for (key, path) in &legacy_meta_files {
+                    if meta_cache.contains_key(key) {
+                        continue;
+                    }
+                    if let Ok(content) = fs::read(path) {
+                        if let Some(etag) = extract_etag_from_meta_bytes(&content) {
+                            meta_cache.insert(key.clone(), etag);
+                        }
+                    }
+                }
+
+                etag_cache_changed = true;
+            }
+
+            let bucket_p = Path::new(&bucket_owned);
+            let bucket_len = bucket_owned.len() + 1;
+            let mut objects: Vec<(String, u64, f64, Option<String>)> = Vec::new();
+
+            if bucket_p.is_dir() {
+                let mut stack = vec![bucket_p.to_path_buf()];
+                while let Some(current) = stack.pop() {
+                    let entries = match fs::read_dir(&current) {
+                        Ok(e) => e,
+                        Err(_) => continue,
+                    };
+                    for entry_result in entries {
+                        let entry = match entry_result {
+                            Ok(e) => e,
+                            Err(_) => continue,
+                        };
+                        let ft = match entry.file_type() {
+                            Ok(ft) => ft,
+                            Err(_) => continue,
+                        };
+                        if ft.is_dir() && !ft.is_symlink() {
+                            let full = entry.path();
+                            let full_str = full.to_string_lossy();
+                            if full_str.len() > bucket_len {
+                                let first_part: &str = if let Some(sep_pos) =
+                                    full_str[bucket_len..].find(|c: char| c == '\\' || c == '/')
+                                {
+                                    &full_str[bucket_len..bucket_len + sep_pos]
+                                } else {
+                                    &full_str[bucket_len..]
+                                };
+                                if INTERNAL_FOLDERS.contains(&first_part) {
+                                    continue;
+                                }
+                            } else if let Some(name) = entry.file_name().to_str() {
+                                if INTERNAL_FOLDERS.contains(&name) {
+                                    continue;
+                                }
+                            }
+                            stack.push(full);
+                        } else if ft.is_file() && !ft.is_symlink() {
+                            let full = entry.path();
+                            let full_str = full.to_string_lossy();
+                            if full_str.len() <= bucket_len {
+                                continue;
+                            }
+                            let rel = &full_str[bucket_len..];
+                            let first_part: &str =
+                                if let Some(sep_pos) = rel.find(|c: char| c == '\\' || c == '/') {
+                                    &rel[..sep_pos]
+                                } else {
+                                    rel
+                                };
+                            if INTERNAL_FOLDERS.contains(&first_part) {
+                                continue;
+                            }
+                            let key = rel.replace('\\', "/");
+                            if let Ok(md) = entry.metadata() {
+                                let size = md.len();
+                                let mtime = md
+                                    .modified()
+                                    .map(system_time_to_epoch)
+                                    .unwrap_or(0.0);
+                                let etag = meta_cache.get(&key).cloned();
+                                objects.push((key, size, mtime, etag));
+                            }
+                        }
+                    }
+                }
+            }
+
+            Ok((meta_cache, objects, etag_cache_changed))
+        })?;
+
+    let (meta_cache, objects, etag_cache_changed) = result;
+
+    let dict = PyDict::new(py);
+
+    let cache_dict = PyDict::new(py);
+    for (k, v) in &meta_cache {
+        cache_dict.set_item(k, v)?;
+    }
+    dict.set_item("etag_cache", cache_dict)?;
+
+    let objects_list = PyList::empty(py);
+    for (key, size, mtime, etag) in &objects {
+        let etag_py: Py<PyAny> = match etag {
+            Some(e) => PyString::new(py, e).into_any().unbind(),
+            None => py.None(),
+        };
+        let tuple = PyTuple::new(py, &[
+            PyString::new(py, key).into_any().unbind(),
+            size.into_pyobject(py)?.into_any().unbind(),
+            mtime.into_pyobject(py)?.into_any().unbind(),
+            etag_py,
+        ])?;
+        objects_list.append(tuple)?;
+    }
+    dict.set_item("objects", objects_list)?;
+    dict.set_item("etag_cache_changed", etag_cache_changed)?;
+
+    Ok(dict.into_any().unbind())
+}

python/myfsio_core/src/streaming.rs

@@ -0,0 +1,112 @@
+use md5::{Digest, Md5};
+use pyo3::exceptions::{PyIOError, PyValueError};
+use pyo3::prelude::*;
+use std::fs::{self, File};
+use std::io::{Read, Write};
+use uuid::Uuid;
+
+const DEFAULT_CHUNK_SIZE: usize = 262144;
+
+#[pyfunction]
+#[pyo3(signature = (stream, tmp_dir, chunk_size=DEFAULT_CHUNK_SIZE))]
+pub fn stream_to_file_with_md5(
+    py: Python<'_>,
+    stream: &Bound<'_, PyAny>,
+    tmp_dir: &str,
+    chunk_size: usize,
+) -> PyResult<(String, String, u64)> {
+    let chunk_size = if chunk_size == 0 {
+        DEFAULT_CHUNK_SIZE
+    } else {
+        chunk_size
+    };
+
+    fs::create_dir_all(tmp_dir)
+        .map_err(|e| PyIOError::new_err(format!("Failed to create tmp dir: {}", e)))?;
+
+    let tmp_name = format!("{}.tmp", Uuid::new_v4().as_hyphenated());
+    let tmp_path_buf = std::path::PathBuf::from(tmp_dir).join(&tmp_name);
+    let tmp_path = tmp_path_buf.to_string_lossy().into_owned();
+
+    let mut file = File::create(&tmp_path)
+        .map_err(|e| PyIOError::new_err(format!("Failed to create temp file: {}", e)))?;
+    let mut hasher = Md5::new();
+    let mut total_bytes: u64 = 0;
+
+    let result: PyResult<()> = (|| {
+        loop {
+            let chunk: Vec<u8> = stream.call_method1("read", (chunk_size,))?.extract()?;
+            if chunk.is_empty() {
+                break;
+            }
+            hasher.update(&chunk);
+            file.write_all(&chunk)
+                .map_err(|e| PyIOError::new_err(format!("Failed to write: {}", e)))?;
+            total_bytes += chunk.len() as u64;
+
+            py.check_signals()?;
+        }
+        file.sync_all()
+            .map_err(|e| PyIOError::new_err(format!("Failed to fsync: {}", e)))?;
+        Ok(())
+    })();
+
+    if let Err(e) = result {
+        drop(file);
+        let _ = fs::remove_file(&tmp_path);
+        return Err(e);
+    }
+
+    drop(file);
+
+    let md5_hex = format!("{:x}", hasher.finalize());
+    Ok((tmp_path, md5_hex, total_bytes))
+}
+
+#[pyfunction]
+pub fn assemble_parts_with_md5(
+    py: Python<'_>,
+    part_paths: Vec<String>,
+    dest_path: &str,
+) -> PyResult<String> {
+    if part_paths.is_empty() {
+        return Err(PyValueError::new_err("No parts to assemble"));
+    }
+
+    let dest = dest_path.to_owned();
+    let parts = part_paths;
+
+    py.detach(move || {
+        if let Some(parent) = std::path::Path::new(&dest).parent() {
+            fs::create_dir_all(parent)
+                .map_err(|e| PyIOError::new_err(format!("Failed to create dest dir: {}", e)))?;
+        }
+
+        let mut target = File::create(&dest)
+            .map_err(|e| PyIOError::new_err(format!("Failed to create dest file: {}", e)))?;
+        let mut hasher = Md5::new();
+        let mut buf = vec![0u8; 1024 * 1024];
+
+        for part_path in &parts {
+            let mut part = File::open(part_path)
+                .map_err(|e| PyIOError::new_err(format!("Failed to open part {}: {}", part_path, e)))?;
+            loop {
+                let n = part
+                    .read(&mut buf)
+                    .map_err(|e| PyIOError::new_err(format!("Failed to read part: {}", e)))?;
+                if n == 0 {
+                    break;
+                }
+                hasher.update(&buf[..n]);
+                target
+                    .write_all(&buf[..n])
+                    .map_err(|e| PyIOError::new_err(format!("Failed to write: {}", e)))?;
+            }
+        }
+
+        target.sync_all()
+            .map_err(|e| PyIOError::new_err(format!("Failed to fsync: {}", e)))?;
+
+        Ok(format!("{:x}", hasher.finalize()))
+    })
+}

python/myfsio_core/src/validation.rs

@@ -0,0 +1,149 @@
+use pyo3::prelude::*;
+use std::sync::LazyLock;
+use unicode_normalization::UnicodeNormalization;
+
+const WINDOWS_RESERVED: &[&str] = &[
+    "CON", "PRN", "AUX", "NUL", "COM0", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7",
+    "COM8", "COM9", "LPT0", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8",
+    "LPT9",
+];
+
+const WINDOWS_ILLEGAL_CHARS: &[char] = &['<', '>', ':', '"', '/', '\\', '|', '?', '*'];
+
+const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"];
+const SYSTEM_ROOT: &str = ".myfsio.sys";
+
+static IP_REGEX: LazyLock<regex::Regex> =
+    LazyLock::new(|| regex::Regex::new(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$").unwrap());
+
+#[pyfunction]
+#[pyo3(signature = (object_key, max_length_bytes=1024, is_windows=false, reserved_prefixes=None))]
+pub fn validate_object_key(
+    object_key: &str,
+    max_length_bytes: usize,
+    is_windows: bool,
+    reserved_prefixes: Option<Vec<String>>,
+) -> PyResult<Option<String>> {
+    if object_key.is_empty() {
+        return Ok(Some("Object key required".to_string()));
+    }
+
+    if object_key.contains('\0') {
+        return Ok(Some("Object key contains null bytes".to_string()));
+    }
+
+    let normalized: String = object_key.nfc().collect();
+
+    if normalized.len() > max_length_bytes {
+        return Ok(Some(format!(
+            "Object key exceeds maximum length of {} bytes",
+            max_length_bytes
+        )));
+    }
+
+    if normalized.starts_with('/') || normalized.starts_with('\\') {
+        return Ok(Some("Object key cannot start with a slash".to_string()));
+    }
+
+    let parts: Vec<&str> = if cfg!(windows) || is_windows {
+        normalized.split(['/', '\\']).collect()
+    } else {
+        normalized.split('/').collect()
+    };
+
+    for part in &parts {
+        if part.is_empty() {
+            continue;
+        }
+
+        if *part == ".." {
+            return Ok(Some(
+                "Object key contains parent directory references".to_string(),
+            ));
+        }
+
+        if *part == "." {
+            return Ok(Some("Object key contains invalid segments".to_string()));
+        }
+
+        if part.chars().any(|c| (c as u32) < 32) {
+            return Ok(Some(
+                "Object key contains control characters".to_string(),
+            ));
+        }
+
+        if is_windows {
+            if part.chars().any(|c| WINDOWS_ILLEGAL_CHARS.contains(&c)) {
+                return Ok(Some(
+                    "Object key contains characters not supported on Windows filesystems"
+                        .to_string(),
+                ));
+            }
+            if part.ends_with(' ') || part.ends_with('.') {
+                return Ok(Some(
+                    "Object key segments cannot end with spaces or periods on Windows".to_string(),
+                ));
+            }
+            let trimmed = part.trim_end_matches(['.', ' ']).to_uppercase();
+            if WINDOWS_RESERVED.contains(&trimmed.as_str()) {
+                return Ok(Some(format!("Invalid filename segment: {}", part)));
+            }
+        }
+    }
+
+    let non_empty_parts: Vec<&str> = parts.iter().filter(|p| !p.is_empty()).copied().collect();
+    if let Some(top) = non_empty_parts.first() {
+        if INTERNAL_FOLDERS.contains(top) || *top == SYSTEM_ROOT {
+            return Ok(Some("Object key uses a reserved prefix".to_string()));
+        }
+
+        if let Some(ref prefixes) = reserved_prefixes {
+            for prefix in prefixes {
+                if *top == prefix.as_str() {
+                    return Ok(Some("Object key uses a reserved prefix".to_string()));
+                }
+            }
+        }
+    }
+
+    Ok(None)
+}
+
+#[pyfunction]
+pub fn validate_bucket_name(bucket_name: &str) -> Option<String> {
+    let len = bucket_name.len();
+    if len < 3 || len > 63 {
+        return Some("Bucket name must be between 3 and 63 characters".to_string());
+    }
+
+    let bytes = bucket_name.as_bytes();
+    if !bytes[0].is_ascii_lowercase() && !bytes[0].is_ascii_digit() {
+        return Some(
+            "Bucket name must start and end with a lowercase letter or digit".to_string(),
+        );
+    }
+    if !bytes[len - 1].is_ascii_lowercase() && !bytes[len - 1].is_ascii_digit() {
+        return Some(
+            "Bucket name must start and end with a lowercase letter or digit".to_string(),
+        );
+    }
+
+    for &b in bytes {
+        if !b.is_ascii_lowercase() && !b.is_ascii_digit() && b != b'.' && b != b'-' {
+            return Some(
+                "Bucket name can only contain lowercase letters, digits, dots, and hyphens"
+                    .to_string(),
+            );
+        }
+    }
+
+    if bucket_name.contains("..") {
+        return Some("Bucket name must not contain consecutive periods".to_string());
+    }
+
+    if IP_REGEX.is_match(bucket_name) {
+        return Some("Bucket name must not be formatted as an IP address".to_string());
+    }
+
+    None
+}