Move index JSON read to Rust for GIL-released parsing (serde_json)

app/storage.py

@@ -1905,6 +1905,8 @@ class ObjectStorage:
 
     def _read_index_entry(self, bucket_name: str, key: Path) -> Optional[Dict[str, Any]]:
         index_path, entry_name = self._index_file_for_key(bucket_name, key)
+        if _HAS_RUST:
+            return _rc.read_index_entry(str(index_path), entry_name)
         if not index_path.exists():
             return None
         try:

myfsio_core/Cargo.toml

@@ -14,6 +14,7 @@ sha2 = "0.10"
 md-5 = "0.10"
 hex = "0.4"
 unicode-normalization = "0.1"
+serde_json = "1"
 regex = "1"
 lru = "0.14"
 parking_lot = "0.12"

myfsio_core/src/lib.rs

@@ -1,4 +1,5 @@
 mod hashing;
+mod metadata;
 mod sigv4;
 mod validation;
 
@@ -25,6 +26,8 @@ mod myfsio_core {
         m.add_function(wrap_pyfunction!(validation::validate_object_key, m)?)?;
         m.add_function(wrap_pyfunction!(validation::validate_bucket_name, m)?)?;
 
+        m.add_function(wrap_pyfunction!(metadata::read_index_entry, m)?)?;
+
         Ok(())
     }
 }

myfsio_core/src/metadata.rs

@@ -0,0 +1,71 @@
+use pyo3::exceptions::PyValueError;
+use pyo3::prelude::*;
+use pyo3::types::{PyDict, PyList, PyString};
+use serde_json::Value;
+use std::fs;
+
+const MAX_DEPTH: u32 = 64;
+
+fn value_to_py(py: Python<'_>, v: &Value, depth: u32) -> PyResult<Py<PyAny>> {
+    if depth > MAX_DEPTH {
+        return Err(PyValueError::new_err("JSON nesting too deep"));
+    }
+    match v {
+        Value::Null => Ok(py.None()),
+        Value::Bool(b) => Ok((*b).into_pyobject(py)?.to_owned().into_any().unbind()),
+        Value::Number(n) => {
+            if let Some(i) = n.as_i64() {
+                Ok(i.into_pyobject(py)?.into_any().unbind())
+            } else if let Some(f) = n.as_f64() {
+                Ok(f.into_pyobject(py)?.into_any().unbind())
+            } else {
+                Ok(py.None())
+            }
+        }
+        Value::String(s) => Ok(PyString::new(py, s).into_any().unbind()),
+        Value::Array(arr) => {
+            let list = PyList::empty(py);
+            for item in arr {
+                list.append(value_to_py(py, item, depth + 1)?)?;
+            }
+            Ok(list.into_any().unbind())
+        }
+        Value::Object(map) => {
+            let dict = PyDict::new(py);
+            for (k, val) in map {
+                dict.set_item(k, value_to_py(py, val, depth + 1)?)?;
+            }
+            Ok(dict.into_any().unbind())
+        }
+    }
+}
+
+#[pyfunction]
+pub fn read_index_entry(
+    py: Python<'_>,
+    path: &str,
+    entry_name: &str,
+) -> PyResult<Option<Py<PyAny>>> {
+    let path_owned = path.to_owned();
+    let entry_owned = entry_name.to_owned();
+
+    let entry: Option<Value> = py.detach(move || -> PyResult<Option<Value>> {
+        let content = match fs::read_to_string(&path_owned) {
+            Ok(c) => c,
+            Err(_) => return Ok(None),
+        };
+        let parsed: Value = match serde_json::from_str(&content) {
+            Ok(v) => v,
+            Err(_) => return Ok(None),
+        };
+        match parsed {
+            Value::Object(mut map) => Ok(map.remove(&entry_owned)),
+            _ => Ok(None),
+        }
+    })?;
+
+    match entry {
+        Some(val) => Ok(Some(value_to_py(py, &val, 0)?)),
+        None => Ok(None),
+    }
+}