Separate Python and Rust into python/ and rust/ with per-stack Dockerfiles

rust/myfsio-engine/crates/myfsio-crypto/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "myfsio-crypto"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+myfsio-common = { path = "../myfsio-common" }
+md-5 = { workspace = true }
+sha2 = { workspace = true }
+hex = { workspace = true }
+aes-gcm = { workspace = true }
+hkdf = { workspace = true }
+thiserror = { workspace = true }
+tokio = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+uuid = { workspace = true }
+chrono = { workspace = true }
+base64 = { workspace = true }
+rand = "0.8"
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
+tempfile = "3"

rust/myfsio-engine/crates/myfsio-crypto/src/aes_gcm.rs

@@ -0,0 +1,238 @@
+use aes_gcm::aead::Aead;
+use aes_gcm::{Aes256Gcm, KeyInit, Nonce};
+use hkdf::Hkdf;
+use sha2::Sha256;
+use std::fs::File;
+use std::io::{Read, Seek, SeekFrom, Write};
+use std::path::Path;
+use thiserror::Error;
+
+const DEFAULT_CHUNK_SIZE: usize = 65536;
+const HEADER_SIZE: usize = 4;
+
+#[derive(Debug, Error)]
+pub enum CryptoError {
+    #[error("IO error: {0}")]
+    Io(#[from] std::io::Error),
+    #[error("Invalid key size: expected 32 bytes, got {0}")]
+    InvalidKeySize(usize),
+    #[error("Invalid nonce size: expected 12 bytes, got {0}")]
+    InvalidNonceSize(usize),
+    #[error("Encryption failed: {0}")]
+    EncryptionFailed(String),
+    #[error("Decryption failed at chunk {0}")]
+    DecryptionFailed(u32),
+    #[error("HKDF expand failed: {0}")]
+    HkdfFailed(String),
+}
+
+fn read_exact_chunk(reader: &mut impl Read, buf: &mut [u8]) -> std::io::Result<usize> {
+    let mut filled = 0;
+    while filled < buf.len() {
+        match reader.read(&mut buf[filled..]) {
+            Ok(0) => break,
+            Ok(n) => filled += n,
+            Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
+            Err(e) => return Err(e),
+        }
+    }
+    Ok(filled)
+}
+
+fn derive_chunk_nonce(base_nonce: &[u8], chunk_index: u32) -> Result<[u8; 12], CryptoError> {
+    let hkdf = Hkdf::<Sha256>::new(Some(base_nonce), b"chunk_nonce");
+    let mut okm = [0u8; 12];
+    hkdf.expand(&chunk_index.to_be_bytes(), &mut okm)
+        .map_err(|e| CryptoError::HkdfFailed(e.to_string()))?;
+    Ok(okm)
+}
+
+pub fn encrypt_stream_chunked(
+    input_path: &Path,
+    output_path: &Path,
+    key: &[u8],
+    base_nonce: &[u8],
+    chunk_size: Option<usize>,
+) -> Result<u32, CryptoError> {
+    if key.len() != 32 {
+        return Err(CryptoError::InvalidKeySize(key.len()));
+    }
+    if base_nonce.len() != 12 {
+        return Err(CryptoError::InvalidNonceSize(base_nonce.len()));
+    }
+
+    let chunk_size = chunk_size.unwrap_or(DEFAULT_CHUNK_SIZE);
+    let key_arr: [u8; 32] = key.try_into().unwrap();
+    let nonce_arr: [u8; 12] = base_nonce.try_into().unwrap();
+    let cipher = Aes256Gcm::new(&key_arr.into());
+
+    let mut infile = File::open(input_path)?;
+    let mut outfile = File::create(output_path)?;
+
+    outfile.write_all(&[0u8; 4])?;
+
+    let mut buf = vec![0u8; chunk_size];
+    let mut chunk_index: u32 = 0;
+
+    loop {
+        let n = read_exact_chunk(&mut infile, &mut buf)?;
+        if n == 0 {
+            break;
+        }
+
+        let nonce_bytes = derive_chunk_nonce(&nonce_arr, chunk_index)?;
+        let nonce = Nonce::from_slice(&nonce_bytes);
+
+        let encrypted = cipher
+            .encrypt(nonce, &buf[..n])
+            .map_err(|e| CryptoError::EncryptionFailed(e.to_string()))?;
+
+        let size = encrypted.len() as u32;
+        outfile.write_all(&size.to_be_bytes())?;
+        outfile.write_all(&encrypted)?;
+
+        chunk_index += 1;
+    }
+
+    outfile.seek(SeekFrom::Start(0))?;
+    outfile.write_all(&chunk_index.to_be_bytes())?;
+
+    Ok(chunk_index)
+}
+
+pub fn decrypt_stream_chunked(
+    input_path: &Path,
+    output_path: &Path,
+    key: &[u8],
+    base_nonce: &[u8],
+) -> Result<u32, CryptoError> {
+    if key.len() != 32 {
+        return Err(CryptoError::InvalidKeySize(key.len()));
+    }
+    if base_nonce.len() != 12 {
+        return Err(CryptoError::InvalidNonceSize(base_nonce.len()));
+    }
+
+    let key_arr: [u8; 32] = key.try_into().unwrap();
+    let nonce_arr: [u8; 12] = base_nonce.try_into().unwrap();
+    let cipher = Aes256Gcm::new(&key_arr.into());
+
+    let mut infile = File::open(input_path)?;
+    let mut outfile = File::create(output_path)?;
+
+    let mut header = [0u8; HEADER_SIZE];
+    infile.read_exact(&mut header)?;
+    let chunk_count = u32::from_be_bytes(header);
+
+    let mut size_buf = [0u8; HEADER_SIZE];
+    for chunk_index in 0..chunk_count {
+        infile.read_exact(&mut size_buf)?;
+        let chunk_size = u32::from_be_bytes(size_buf) as usize;
+
+        let mut encrypted = vec![0u8; chunk_size];
+        infile.read_exact(&mut encrypted)?;
+
+        let nonce_bytes = derive_chunk_nonce(&nonce_arr, chunk_index)?;
+        let nonce = Nonce::from_slice(&nonce_bytes);
+
+        let decrypted = cipher
+            .decrypt(nonce, encrypted.as_ref())
+            .map_err(|_| CryptoError::DecryptionFailed(chunk_index))?;
+
+        outfile.write_all(&decrypted)?;
+    }
+
+    Ok(chunk_count)
+}
+
+pub async fn encrypt_stream_chunked_async(
+    input_path: &Path,
+    output_path: &Path,
+    key: &[u8],
+    base_nonce: &[u8],
+    chunk_size: Option<usize>,
+) -> Result<u32, CryptoError> {
+    let input_path = input_path.to_owned();
+    let output_path = output_path.to_owned();
+    let key = key.to_vec();
+    let base_nonce = base_nonce.to_vec();
+    tokio::task::spawn_blocking(move || {
+        encrypt_stream_chunked(&input_path, &output_path, &key, &base_nonce, chunk_size)
+    })
+    .await
+    .map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))?
+}
+
+pub async fn decrypt_stream_chunked_async(
+    input_path: &Path,
+    output_path: &Path,
+    key: &[u8],
+    base_nonce: &[u8],
+) -> Result<u32, CryptoError> {
+    let input_path = input_path.to_owned();
+    let output_path = output_path.to_owned();
+    let key = key.to_vec();
+    let base_nonce = base_nonce.to_vec();
+    tokio::task::spawn_blocking(move || {
+        decrypt_stream_chunked(&input_path, &output_path, &key, &base_nonce)
+    })
+    .await
+    .map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))?
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::Write as IoWrite;
+
+    #[test]
+    fn test_encrypt_decrypt_roundtrip() {
+        let dir = tempfile::tempdir().unwrap();
+        let input = dir.path().join("input.bin");
+        let encrypted = dir.path().join("encrypted.bin");
+        let decrypted = dir.path().join("decrypted.bin");
+
+        let data = b"Hello, this is a test of AES-256-GCM chunked encryption!";
+        std::fs::File::create(&input).unwrap().write_all(data).unwrap();
+
+        let key = [0x42u8; 32];
+        let nonce = [0x01u8; 12];
+
+        let chunks = encrypt_stream_chunked(&input, &encrypted, &key, &nonce, Some(16)).unwrap();
+        assert!(chunks > 0);
+
+        let chunks2 = decrypt_stream_chunked(&encrypted, &decrypted, &key, &nonce).unwrap();
+        assert_eq!(chunks, chunks2);
+
+        let result = std::fs::read(&decrypted).unwrap();
+        assert_eq!(result, data);
+    }
+
+    #[test]
+    fn test_invalid_key_size() {
+        let dir = tempfile::tempdir().unwrap();
+        let input = dir.path().join("input.bin");
+        std::fs::File::create(&input).unwrap().write_all(b"test").unwrap();
+
+        let result = encrypt_stream_chunked(&input, &dir.path().join("out"), &[0u8; 16], &[0u8; 12], None);
+        assert!(matches!(result, Err(CryptoError::InvalidKeySize(16))));
+    }
+
+    #[test]
+    fn test_wrong_key_fails_decrypt() {
+        let dir = tempfile::tempdir().unwrap();
+        let input = dir.path().join("input.bin");
+        let encrypted = dir.path().join("encrypted.bin");
+        let decrypted = dir.path().join("decrypted.bin");
+
+        std::fs::File::create(&input).unwrap().write_all(b"secret data").unwrap();
+
+        let key = [0x42u8; 32];
+        let nonce = [0x01u8; 12];
+        encrypt_stream_chunked(&input, &encrypted, &key, &nonce, None).unwrap();
+
+        let wrong_key = [0x43u8; 32];
+        let result = decrypt_stream_chunked(&encrypted, &decrypted, &wrong_key, &nonce);
+        assert!(matches!(result, Err(CryptoError::DecryptionFailed(_))));
+    }
+}

rust/myfsio-engine/crates/myfsio-crypto/src/encryption.rs

@@ -0,0 +1,375 @@
+use base64::engine::general_purpose::STANDARD as B64;
+use base64::Engine;
+use rand::RngCore;
+use std::collections::HashMap;
+use std::path::Path;
+
+use crate::aes_gcm::{
+    encrypt_stream_chunked, decrypt_stream_chunked, CryptoError,
+};
+use crate::kms::KmsService;
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum SseAlgorithm {
+    Aes256,
+    AwsKms,
+    CustomerProvided,
+}
+
+impl SseAlgorithm {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            SseAlgorithm::Aes256 => "AES256",
+            SseAlgorithm::AwsKms => "aws:kms",
+            SseAlgorithm::CustomerProvided => "AES256",
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct EncryptionContext {
+    pub algorithm: SseAlgorithm,
+    pub kms_key_id: Option<String>,
+    pub customer_key: Option<Vec<u8>>,
+}
+
+#[derive(Debug, Clone)]
+pub struct EncryptionMetadata {
+    pub algorithm: String,
+    pub nonce: String,
+    pub encrypted_data_key: Option<String>,
+    pub kms_key_id: Option<String>,
+}
+
+impl EncryptionMetadata {
+    pub fn to_metadata_map(&self) -> HashMap<String, String> {
+        let mut map = HashMap::new();
+        map.insert(
+            "x-amz-server-side-encryption".to_string(),
+            self.algorithm.clone(),
+        );
+        map.insert("x-amz-encryption-nonce".to_string(), self.nonce.clone());
+        if let Some(ref dk) = self.encrypted_data_key {
+            map.insert("x-amz-encrypted-data-key".to_string(), dk.clone());
+        }
+        if let Some(ref kid) = self.kms_key_id {
+            map.insert("x-amz-encryption-key-id".to_string(), kid.clone());
+        }
+        map
+    }
+
+    pub fn from_metadata(meta: &HashMap<String, String>) -> Option<Self> {
+        let algorithm = meta.get("x-amz-server-side-encryption")?;
+        let nonce = meta.get("x-amz-encryption-nonce")?;
+        Some(Self {
+            algorithm: algorithm.clone(),
+            nonce: nonce.clone(),
+            encrypted_data_key: meta.get("x-amz-encrypted-data-key").cloned(),
+            kms_key_id: meta.get("x-amz-encryption-key-id").cloned(),
+        })
+    }
+
+    pub fn is_encrypted(meta: &HashMap<String, String>) -> bool {
+        meta.contains_key("x-amz-server-side-encryption")
+    }
+
+    pub fn clean_metadata(meta: &mut HashMap<String, String>) {
+        meta.remove("x-amz-server-side-encryption");
+        meta.remove("x-amz-encryption-nonce");
+        meta.remove("x-amz-encrypted-data-key");
+        meta.remove("x-amz-encryption-key-id");
+    }
+}
+
+pub struct EncryptionService {
+    master_key: [u8; 32],
+    kms: Option<std::sync::Arc<KmsService>>,
+}
+
+impl EncryptionService {
+    pub fn new(master_key: [u8; 32], kms: Option<std::sync::Arc<KmsService>>) -> Self {
+        Self { master_key, kms }
+    }
+
+    pub fn generate_data_key(&self) -> ([u8; 32], [u8; 12]) {
+        let mut data_key = [0u8; 32];
+        let mut nonce = [0u8; 12];
+        rand::thread_rng().fill_bytes(&mut data_key);
+        rand::thread_rng().fill_bytes(&mut nonce);
+        (data_key, nonce)
+    }
+
+    pub fn wrap_data_key(&self, data_key: &[u8; 32]) -> Result<String, CryptoError> {
+        use aes_gcm::aead::Aead;
+        use aes_gcm::{Aes256Gcm, KeyInit, Nonce};
+
+        let cipher = Aes256Gcm::new((&self.master_key).into());
+        let mut nonce_bytes = [0u8; 12];
+        rand::thread_rng().fill_bytes(&mut nonce_bytes);
+        let nonce = Nonce::from_slice(&nonce_bytes);
+
+        let encrypted = cipher
+            .encrypt(nonce, data_key.as_slice())
+            .map_err(|e| CryptoError::EncryptionFailed(e.to_string()))?;
+
+        let mut combined = Vec::with_capacity(12 + encrypted.len());
+        combined.extend_from_slice(&nonce_bytes);
+        combined.extend_from_slice(&encrypted);
+        Ok(B64.encode(&combined))
+    }
+
+    pub fn unwrap_data_key(&self, wrapped_b64: &str) -> Result<[u8; 32], CryptoError> {
+        use aes_gcm::aead::Aead;
+        use aes_gcm::{Aes256Gcm, KeyInit, Nonce};
+
+        let combined = B64.decode(wrapped_b64).map_err(|e| {
+            CryptoError::EncryptionFailed(format!("Bad wrapped key encoding: {}", e))
+        })?;
+        if combined.len() < 12 {
+            return Err(CryptoError::EncryptionFailed(
+                "Wrapped key too short".to_string(),
+            ));
+        }
+
+        let (nonce_bytes, ciphertext) = combined.split_at(12);
+        let cipher = Aes256Gcm::new((&self.master_key).into());
+        let nonce = Nonce::from_slice(nonce_bytes);
+
+        let plaintext = cipher
+            .decrypt(nonce, ciphertext)
+            .map_err(|_| CryptoError::DecryptionFailed(0))?;
+
+        if plaintext.len() != 32 {
+            return Err(CryptoError::InvalidKeySize(plaintext.len()));
+        }
+        let mut key = [0u8; 32];
+        key.copy_from_slice(&plaintext);
+        Ok(key)
+    }
+
+    pub async fn encrypt_object(
+        &self,
+        input_path: &Path,
+        output_path: &Path,
+        ctx: &EncryptionContext,
+    ) -> Result<EncryptionMetadata, CryptoError> {
+        let (data_key, nonce) = self.generate_data_key();
+
+        let (encrypted_data_key, kms_key_id) = match ctx.algorithm {
+            SseAlgorithm::Aes256 => {
+                let wrapped = self.wrap_data_key(&data_key)?;
+                (Some(wrapped), None)
+            }
+            SseAlgorithm::AwsKms => {
+                let kms = self
+                    .kms
+                    .as_ref()
+                    .ok_or_else(|| CryptoError::EncryptionFailed("KMS not available".into()))?;
+                let kid = ctx
+                    .kms_key_id
+                    .as_ref()
+                    .ok_or_else(|| CryptoError::EncryptionFailed("No KMS key ID".into()))?;
+                let ciphertext = kms.encrypt_data(kid, &data_key).await?;
+                (Some(B64.encode(&ciphertext)), Some(kid.clone()))
+            }
+            SseAlgorithm::CustomerProvided => {
+                (None, None)
+            }
+        };
+
+        let actual_key = if ctx.algorithm == SseAlgorithm::CustomerProvided {
+            let ck = ctx.customer_key.as_ref().ok_or_else(|| {
+                CryptoError::EncryptionFailed("No customer key provided".into())
+            })?;
+            if ck.len() != 32 {
+                return Err(CryptoError::InvalidKeySize(ck.len()));
+            }
+            let mut k = [0u8; 32];
+            k.copy_from_slice(ck);
+            k
+        } else {
+            data_key
+        };
+
+        let ip = input_path.to_owned();
+        let op = output_path.to_owned();
+        let ak = actual_key;
+        let n = nonce;
+        tokio::task::spawn_blocking(move || {
+            encrypt_stream_chunked(&ip, &op, &ak, &n, None)
+        })
+        .await
+        .map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))??;
+
+        Ok(EncryptionMetadata {
+            algorithm: ctx.algorithm.as_str().to_string(),
+            nonce: B64.encode(nonce),
+            encrypted_data_key,
+            kms_key_id,
+        })
+    }
+
+    pub async fn decrypt_object(
+        &self,
+        input_path: &Path,
+        output_path: &Path,
+        enc_meta: &EncryptionMetadata,
+        customer_key: Option<&[u8]>,
+    ) -> Result<(), CryptoError> {
+        let nonce_bytes = B64.decode(&enc_meta.nonce).map_err(|e| {
+            CryptoError::EncryptionFailed(format!("Bad nonce encoding: {}", e))
+        })?;
+        if nonce_bytes.len() != 12 {
+            return Err(CryptoError::InvalidNonceSize(nonce_bytes.len()));
+        }
+
+        let data_key: [u8; 32] = if let Some(ck) = customer_key {
+            if ck.len() != 32 {
+                return Err(CryptoError::InvalidKeySize(ck.len()));
+            }
+            let mut k = [0u8; 32];
+            k.copy_from_slice(ck);
+            k
+        } else if enc_meta.algorithm == "aws:kms" {
+            let kms = self
+                .kms
+                .as_ref()
+                .ok_or_else(|| CryptoError::EncryptionFailed("KMS not available".into()))?;
+            let kid = enc_meta
+                .kms_key_id
+                .as_ref()
+                .ok_or_else(|| CryptoError::EncryptionFailed("No KMS key ID in metadata".into()))?;
+            let encrypted_dk = enc_meta.encrypted_data_key.as_ref().ok_or_else(|| {
+                CryptoError::EncryptionFailed("No encrypted data key in metadata".into())
+            })?;
+            let ct = B64.decode(encrypted_dk).map_err(|e| {
+                CryptoError::EncryptionFailed(format!("Bad data key encoding: {}", e))
+            })?;
+            let dk = kms.decrypt_data(kid, &ct).await?;
+            if dk.len() != 32 {
+                return Err(CryptoError::InvalidKeySize(dk.len()));
+            }
+            let mut k = [0u8; 32];
+            k.copy_from_slice(&dk);
+            k
+        } else {
+            let wrapped = enc_meta.encrypted_data_key.as_ref().ok_or_else(|| {
+                CryptoError::EncryptionFailed("No encrypted data key in metadata".into())
+            })?;
+            self.unwrap_data_key(wrapped)?
+        };
+
+        let ip = input_path.to_owned();
+        let op = output_path.to_owned();
+        let nb: [u8; 12] = nonce_bytes.try_into().unwrap();
+        tokio::task::spawn_blocking(move || {
+            decrypt_stream_chunked(&ip, &op, &data_key, &nb)
+        })
+        .await
+        .map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))??;
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::Write;
+
+    fn test_master_key() -> [u8; 32] {
+        [0x42u8; 32]
+    }
+
+    #[test]
+    fn test_wrap_unwrap_data_key() {
+        let svc = EncryptionService::new(test_master_key(), None);
+        let dk = [0xAAu8; 32];
+        let wrapped = svc.wrap_data_key(&dk).unwrap();
+        let unwrapped = svc.unwrap_data_key(&wrapped).unwrap();
+        assert_eq!(dk, unwrapped);
+    }
+
+    #[tokio::test]
+    async fn test_encrypt_decrypt_object_sse_s3() {
+        let dir = tempfile::tempdir().unwrap();
+        let input = dir.path().join("plain.bin");
+        let encrypted = dir.path().join("enc.bin");
+        let decrypted = dir.path().join("dec.bin");
+
+        let data = b"SSE-S3 encrypted content for testing!";
+        std::fs::File::create(&input).unwrap().write_all(data).unwrap();
+
+        let svc = EncryptionService::new(test_master_key(), None);
+
+        let ctx = EncryptionContext {
+            algorithm: SseAlgorithm::Aes256,
+            kms_key_id: None,
+            customer_key: None,
+        };
+
+        let meta = svc.encrypt_object(&input, &encrypted, &ctx).await.unwrap();
+        assert_eq!(meta.algorithm, "AES256");
+        assert!(meta.encrypted_data_key.is_some());
+
+        svc.decrypt_object(&encrypted, &decrypted, &meta, None)
+            .await
+            .unwrap();
+
+        let result = std::fs::read(&decrypted).unwrap();
+        assert_eq!(result, data);
+    }
+
+    #[tokio::test]
+    async fn test_encrypt_decrypt_object_sse_c() {
+        let dir = tempfile::tempdir().unwrap();
+        let input = dir.path().join("plain.bin");
+        let encrypted = dir.path().join("enc.bin");
+        let decrypted = dir.path().join("dec.bin");
+
+        let data = b"SSE-C encrypted content!";
+        std::fs::File::create(&input).unwrap().write_all(data).unwrap();
+
+        let customer_key = [0xBBu8; 32];
+        let svc = EncryptionService::new(test_master_key(), None);
+
+        let ctx = EncryptionContext {
+            algorithm: SseAlgorithm::CustomerProvided,
+            kms_key_id: None,
+            customer_key: Some(customer_key.to_vec()),
+        };
+
+        let meta = svc.encrypt_object(&input, &encrypted, &ctx).await.unwrap();
+        assert!(meta.encrypted_data_key.is_none());
+
+        svc.decrypt_object(&encrypted, &decrypted, &meta, Some(&customer_key))
+            .await
+            .unwrap();
+
+        let result = std::fs::read(&decrypted).unwrap();
+        assert_eq!(result, data);
+    }
+
+    #[test]
+    fn test_encryption_metadata_roundtrip() {
+        let meta = EncryptionMetadata {
+            algorithm: "AES256".to_string(),
+            nonce: "dGVzdG5vbmNlMTI=".to_string(),
+            encrypted_data_key: Some("c29tZWtleQ==".to_string()),
+            kms_key_id: None,
+        };
+        let map = meta.to_metadata_map();
+        let restored = EncryptionMetadata::from_metadata(&map).unwrap();
+        assert_eq!(restored.algorithm, "AES256");
+        assert_eq!(restored.nonce, meta.nonce);
+        assert_eq!(restored.encrypted_data_key, meta.encrypted_data_key);
+    }
+
+    #[test]
+    fn test_is_encrypted() {
+        let mut meta = HashMap::new();
+        assert!(!EncryptionMetadata::is_encrypted(&meta));
+        meta.insert("x-amz-server-side-encryption".to_string(), "AES256".to_string());
+        assert!(EncryptionMetadata::is_encrypted(&meta));
+    }
+}

rust/myfsio-engine/crates/myfsio-crypto/src/hashing.rs

@@ -0,0 +1,132 @@
+use md5::{Digest, Md5};
+use sha2::Sha256;
+use std::io::Read;
+use std::path::Path;
+
+const CHUNK_SIZE: usize = 65536;
+
+pub fn md5_file(path: &Path) -> std::io::Result<String> {
+    let mut file = std::fs::File::open(path)?;
+    let mut hasher = Md5::new();
+    let mut buf = vec![0u8; CHUNK_SIZE];
+    loop {
+        let n = file.read(&mut buf)?;
+        if n == 0 {
+            break;
+        }
+        hasher.update(&buf[..n]);
+    }
+    Ok(format!("{:x}", hasher.finalize()))
+}
+
+pub fn md5_bytes(data: &[u8]) -> String {
+    let mut hasher = Md5::new();
+    hasher.update(data);
+    format!("{:x}", hasher.finalize())
+}
+
+pub fn sha256_file(path: &Path) -> std::io::Result<String> {
+    let mut file = std::fs::File::open(path)?;
+    let mut hasher = Sha256::new();
+    let mut buf = vec![0u8; CHUNK_SIZE];
+    loop {
+        let n = file.read(&mut buf)?;
+        if n == 0 {
+            break;
+        }
+        hasher.update(&buf[..n]);
+    }
+    Ok(format!("{:x}", hasher.finalize()))
+}
+
+pub fn sha256_bytes(data: &[u8]) -> String {
+    let mut hasher = Sha256::new();
+    hasher.update(data);
+    format!("{:x}", hasher.finalize())
+}
+
+pub fn md5_sha256_file(path: &Path) -> std::io::Result<(String, String)> {
+    let mut file = std::fs::File::open(path)?;
+    let mut md5_hasher = Md5::new();
+    let mut sha_hasher = Sha256::new();
+    let mut buf = vec![0u8; CHUNK_SIZE];
+    loop {
+        let n = file.read(&mut buf)?;
+        if n == 0 {
+            break;
+        }
+        md5_hasher.update(&buf[..n]);
+        sha_hasher.update(&buf[..n]);
+    }
+    Ok((
+        format!("{:x}", md5_hasher.finalize()),
+        format!("{:x}", sha_hasher.finalize()),
+    ))
+}
+
+pub async fn md5_file_async(path: &Path) -> std::io::Result<String> {
+    let path = path.to_owned();
+    tokio::task::spawn_blocking(move || md5_file(&path))
+        .await
+        .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?
+}
+
+pub async fn sha256_file_async(path: &Path) -> std::io::Result<String> {
+    let path = path.to_owned();
+    tokio::task::spawn_blocking(move || sha256_file(&path))
+        .await
+        .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?
+}
+
+pub async fn md5_sha256_file_async(path: &Path) -> std::io::Result<(String, String)> {
+    let path = path.to_owned();
+    tokio::task::spawn_blocking(move || md5_sha256_file(&path))
+        .await
+        .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::Write;
+
+    #[test]
+    fn test_md5_bytes() {
+        assert_eq!(md5_bytes(b""), "d41d8cd98f00b204e9800998ecf8427e");
+        assert_eq!(md5_bytes(b"hello"), "5d41402abc4b2a76b9719d911017c592");
+    }
+
+    #[test]
+    fn test_sha256_bytes() {
+        let hash = sha256_bytes(b"hello");
+        assert_eq!(hash, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824");
+    }
+
+    #[test]
+    fn test_md5_file() {
+        let mut tmp = tempfile::NamedTempFile::new().unwrap();
+        tmp.write_all(b"hello").unwrap();
+        tmp.flush().unwrap();
+        let hash = md5_file(tmp.path()).unwrap();
+        assert_eq!(hash, "5d41402abc4b2a76b9719d911017c592");
+    }
+
+    #[test]
+    fn test_md5_sha256_file() {
+        let mut tmp = tempfile::NamedTempFile::new().unwrap();
+        tmp.write_all(b"hello").unwrap();
+        tmp.flush().unwrap();
+        let (md5, sha) = md5_sha256_file(tmp.path()).unwrap();
+        assert_eq!(md5, "5d41402abc4b2a76b9719d911017c592");
+        assert_eq!(sha, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824");
+    }
+
+    #[tokio::test]
+    async fn test_md5_file_async() {
+        let mut tmp = tempfile::NamedTempFile::new().unwrap();
+        tmp.write_all(b"hello").unwrap();
+        tmp.flush().unwrap();
+        let hash = md5_file_async(tmp.path()).await.unwrap();
+        assert_eq!(hash, "5d41402abc4b2a76b9719d911017c592");
+    }
+}

rust/myfsio-engine/crates/myfsio-crypto/src/kms.rs

@@ -0,0 +1,453 @@
+use aes_gcm::aead::Aead;
+use aes_gcm::{Aes256Gcm, KeyInit, Nonce};
+use base64::engine::general_purpose::STANDARD as B64;
+use base64::Engine;
+use chrono::{DateTime, Utc};
+use rand::RngCore;
+use serde::{Deserialize, Serialize};
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+use crate::aes_gcm::CryptoError;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct KmsKey {
+    #[serde(rename = "KeyId")]
+    pub key_id: String,
+    #[serde(rename = "Arn")]
+    pub arn: String,
+    #[serde(rename = "Description")]
+    pub description: String,
+    #[serde(rename = "CreationDate")]
+    pub creation_date: DateTime<Utc>,
+    #[serde(rename = "Enabled")]
+    pub enabled: bool,
+    #[serde(rename = "KeyState")]
+    pub key_state: String,
+    #[serde(rename = "KeyUsage")]
+    pub key_usage: String,
+    #[serde(rename = "KeySpec")]
+    pub key_spec: String,
+    #[serde(rename = "EncryptedKeyMaterial")]
+    pub encrypted_key_material: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct KmsStore {
+    keys: Vec<KmsKey>,
+}
+
+pub struct KmsService {
+    keys_path: PathBuf,
+    master_key: Arc<RwLock<[u8; 32]>>,
+    keys: Arc<RwLock<Vec<KmsKey>>>,
+}
+
+impl KmsService {
+    pub async fn new(keys_dir: &Path) -> Result<Self, CryptoError> {
+        std::fs::create_dir_all(keys_dir).map_err(CryptoError::Io)?;
+
+        let keys_path = keys_dir.join("kms_keys.json");
+
+        let master_key = Self::load_or_create_master_key(&keys_dir.join("kms_master.key"))?;
+
+        let keys = if keys_path.exists() {
+            let data = std::fs::read_to_string(&keys_path).map_err(CryptoError::Io)?;
+            let store: KmsStore = serde_json::from_str(&data)
+                .map_err(|e| CryptoError::EncryptionFailed(format!("Bad KMS store: {}", e)))?;
+            store.keys
+        } else {
+            Vec::new()
+        };
+
+        Ok(Self {
+            keys_path,
+            master_key: Arc::new(RwLock::new(master_key)),
+            keys: Arc::new(RwLock::new(keys)),
+        })
+    }
+
+    fn load_or_create_master_key(path: &Path) -> Result<[u8; 32], CryptoError> {
+        if path.exists() {
+            let encoded = std::fs::read_to_string(path).map_err(CryptoError::Io)?;
+            let decoded = B64.decode(encoded.trim()).map_err(|e| {
+                CryptoError::EncryptionFailed(format!("Bad master key encoding: {}", e))
+            })?;
+            if decoded.len() != 32 {
+                return Err(CryptoError::InvalidKeySize(decoded.len()));
+            }
+            let mut key = [0u8; 32];
+            key.copy_from_slice(&decoded);
+            Ok(key)
+        } else {
+            let mut key = [0u8; 32];
+            rand::thread_rng().fill_bytes(&mut key);
+            let encoded = B64.encode(key);
+            std::fs::write(path, &encoded).map_err(CryptoError::Io)?;
+            Ok(key)
+        }
+    }
+
+    fn encrypt_key_material(
+        master_key: &[u8; 32],
+        plaintext_key: &[u8],
+    ) -> Result<String, CryptoError> {
+        let cipher = Aes256Gcm::new(master_key.into());
+        let mut nonce_bytes = [0u8; 12];
+        rand::thread_rng().fill_bytes(&mut nonce_bytes);
+        let nonce = Nonce::from_slice(&nonce_bytes);
+
+        let ciphertext = cipher
+            .encrypt(nonce, plaintext_key)
+            .map_err(|e| CryptoError::EncryptionFailed(e.to_string()))?;
+
+        let mut combined = Vec::with_capacity(12 + ciphertext.len());
+        combined.extend_from_slice(&nonce_bytes);
+        combined.extend_from_slice(&ciphertext);
+        Ok(B64.encode(&combined))
+    }
+
+    fn decrypt_key_material(
+        master_key: &[u8; 32],
+        encrypted_b64: &str,
+    ) -> Result<Vec<u8>, CryptoError> {
+        let combined = B64.decode(encrypted_b64).map_err(|e| {
+            CryptoError::EncryptionFailed(format!("Bad key material encoding: {}", e))
+        })?;
+        if combined.len() < 12 {
+            return Err(CryptoError::EncryptionFailed(
+                "Encrypted key material too short".to_string(),
+            ));
+        }
+
+        let (nonce_bytes, ciphertext) = combined.split_at(12);
+        let cipher = Aes256Gcm::new(master_key.into());
+        let nonce = Nonce::from_slice(nonce_bytes);
+
+        cipher
+            .decrypt(nonce, ciphertext)
+            .map_err(|_| CryptoError::DecryptionFailed(0))
+    }
+
+    async fn save(&self) -> Result<(), CryptoError> {
+        let keys = self.keys.read().await;
+        let store = KmsStore {
+            keys: keys.clone(),
+        };
+        let json = serde_json::to_string_pretty(&store)
+            .map_err(|e| CryptoError::EncryptionFailed(e.to_string()))?;
+        std::fs::write(&self.keys_path, json).map_err(CryptoError::Io)?;
+        Ok(())
+    }
+
+    pub async fn create_key(&self, description: &str) -> Result<KmsKey, CryptoError> {
+        let key_id = uuid::Uuid::new_v4().to_string();
+        let arn = format!("arn:aws:kms:local:000000000000:key/{}", key_id);
+
+        let mut plaintext_key = [0u8; 32];
+        rand::thread_rng().fill_bytes(&mut plaintext_key);
+
+        let master = self.master_key.read().await;
+        let encrypted = Self::encrypt_key_material(&master, &plaintext_key)?;
+
+        let kms_key = KmsKey {
+            key_id: key_id.clone(),
+            arn,
+            description: description.to_string(),
+            creation_date: Utc::now(),
+            enabled: true,
+            key_state: "Enabled".to_string(),
+            key_usage: "ENCRYPT_DECRYPT".to_string(),
+            key_spec: "SYMMETRIC_DEFAULT".to_string(),
+            encrypted_key_material: encrypted,
+        };
+
+        self.keys.write().await.push(kms_key.clone());
+        self.save().await?;
+        Ok(kms_key)
+    }
+
+    pub async fn list_keys(&self) -> Vec<KmsKey> {
+        self.keys.read().await.clone()
+    }
+
+    pub async fn get_key(&self, key_id: &str) -> Option<KmsKey> {
+        let keys = self.keys.read().await;
+        keys.iter()
+            .find(|k| k.key_id == key_id || k.arn == key_id)
+            .cloned()
+    }
+
+    pub async fn delete_key(&self, key_id: &str) -> Result<bool, CryptoError> {
+        let mut keys = self.keys.write().await;
+        let len_before = keys.len();
+        keys.retain(|k| k.key_id != key_id && k.arn != key_id);
+        let removed = keys.len() < len_before;
+        drop(keys);
+        if removed {
+            self.save().await?;
+        }
+        Ok(removed)
+    }
+
+    pub async fn enable_key(&self, key_id: &str) -> Result<bool, CryptoError> {
+        let mut keys = self.keys.write().await;
+        if let Some(key) = keys.iter_mut().find(|k| k.key_id == key_id) {
+            key.enabled = true;
+            key.key_state = "Enabled".to_string();
+            drop(keys);
+            self.save().await?;
+            Ok(true)
+        } else {
+            Ok(false)
+        }
+    }
+
+    pub async fn disable_key(&self, key_id: &str) -> Result<bool, CryptoError> {
+        let mut keys = self.keys.write().await;
+        if let Some(key) = keys.iter_mut().find(|k| k.key_id == key_id) {
+            key.enabled = false;
+            key.key_state = "Disabled".to_string();
+            drop(keys);
+            self.save().await?;
+            Ok(true)
+        } else {
+            Ok(false)
+        }
+    }
+
+    pub async fn decrypt_data_key(&self, key_id: &str) -> Result<Vec<u8>, CryptoError> {
+        let keys = self.keys.read().await;
+        let key = keys
+            .iter()
+            .find(|k| k.key_id == key_id || k.arn == key_id)
+            .ok_or_else(|| CryptoError::EncryptionFailed("KMS key not found".to_string()))?;
+
+        if !key.enabled {
+            return Err(CryptoError::EncryptionFailed(
+                "KMS key is disabled".to_string(),
+            ));
+        }
+
+        let master = self.master_key.read().await;
+        Self::decrypt_key_material(&master, &key.encrypted_key_material)
+    }
+
+    pub async fn encrypt_data(
+        &self,
+        key_id: &str,
+        plaintext: &[u8],
+    ) -> Result<Vec<u8>, CryptoError> {
+        let data_key = self.decrypt_data_key(key_id).await?;
+        if data_key.len() != 32 {
+            return Err(CryptoError::InvalidKeySize(data_key.len()));
+        }
+
+        let key_arr: [u8; 32] = data_key.try_into().unwrap();
+        let cipher = Aes256Gcm::new(&key_arr.into());
+        let mut nonce_bytes = [0u8; 12];
+        rand::thread_rng().fill_bytes(&mut nonce_bytes);
+        let nonce = Nonce::from_slice(&nonce_bytes);
+
+        let ciphertext = cipher
+            .encrypt(nonce, plaintext)
+            .map_err(|e| CryptoError::EncryptionFailed(e.to_string()))?;
+
+        let mut result = Vec::with_capacity(12 + ciphertext.len());
+        result.extend_from_slice(&nonce_bytes);
+        result.extend_from_slice(&ciphertext);
+        Ok(result)
+    }
+
+    pub async fn decrypt_data(
+        &self,
+        key_id: &str,
+        ciphertext: &[u8],
+    ) -> Result<Vec<u8>, CryptoError> {
+        if ciphertext.len() < 12 {
+            return Err(CryptoError::EncryptionFailed(
+                "Ciphertext too short".to_string(),
+            ));
+        }
+
+        let data_key = self.decrypt_data_key(key_id).await?;
+        if data_key.len() != 32 {
+            return Err(CryptoError::InvalidKeySize(data_key.len()));
+        }
+
+        let key_arr: [u8; 32] = data_key.try_into().unwrap();
+        let (nonce_bytes, ct) = ciphertext.split_at(12);
+        let cipher = Aes256Gcm::new(&key_arr.into());
+        let nonce = Nonce::from_slice(nonce_bytes);
+
+        cipher
+            .decrypt(nonce, ct)
+            .map_err(|_| CryptoError::DecryptionFailed(0))
+    }
+
+    pub async fn generate_data_key(
+        &self,
+        key_id: &str,
+        num_bytes: usize,
+    ) -> Result<(Vec<u8>, Vec<u8>), CryptoError> {
+        let kms_key = self.decrypt_data_key(key_id).await?;
+        if kms_key.len() != 32 {
+            return Err(CryptoError::InvalidKeySize(kms_key.len()));
+        }
+
+        let mut plaintext_key = vec![0u8; num_bytes];
+        rand::thread_rng().fill_bytes(&mut plaintext_key);
+
+        let key_arr: [u8; 32] = kms_key.try_into().unwrap();
+        let cipher = Aes256Gcm::new(&key_arr.into());
+        let mut nonce_bytes = [0u8; 12];
+        rand::thread_rng().fill_bytes(&mut nonce_bytes);
+        let nonce = Nonce::from_slice(&nonce_bytes);
+
+        let encrypted = cipher
+            .encrypt(nonce, plaintext_key.as_slice())
+            .map_err(|e| CryptoError::EncryptionFailed(e.to_string()))?;
+
+        let mut wrapped = Vec::with_capacity(12 + encrypted.len());
+        wrapped.extend_from_slice(&nonce_bytes);
+        wrapped.extend_from_slice(&encrypted);
+
+        Ok((plaintext_key, wrapped))
+    }
+}
+
+pub async fn load_or_create_master_key(keys_dir: &Path) -> Result<[u8; 32], CryptoError> {
+    std::fs::create_dir_all(keys_dir).map_err(CryptoError::Io)?;
+    let path = keys_dir.join("master.key");
+
+    if path.exists() {
+        let encoded = std::fs::read_to_string(&path).map_err(CryptoError::Io)?;
+        let decoded = B64.decode(encoded.trim()).map_err(|e| {
+            CryptoError::EncryptionFailed(format!("Bad master key encoding: {}", e))
+        })?;
+        if decoded.len() != 32 {
+            return Err(CryptoError::InvalidKeySize(decoded.len()));
+        }
+        let mut key = [0u8; 32];
+        key.copy_from_slice(&decoded);
+        Ok(key)
+    } else {
+        let mut key = [0u8; 32];
+        rand::thread_rng().fill_bytes(&mut key);
+        let encoded = B64.encode(key);
+        std::fs::write(&path, &encoded).map_err(CryptoError::Io)?;
+        Ok(key)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_create_and_list_keys() {
+        let dir = tempfile::tempdir().unwrap();
+        let kms = KmsService::new(dir.path()).await.unwrap();
+
+        let key = kms.create_key("test key").await.unwrap();
+        assert!(key.enabled);
+        assert_eq!(key.description, "test key");
+        assert!(key.key_id.len() > 0);
+
+        let keys = kms.list_keys().await;
+        assert_eq!(keys.len(), 1);
+        assert_eq!(keys[0].key_id, key.key_id);
+    }
+
+    #[tokio::test]
+    async fn test_enable_disable_key() {
+        let dir = tempfile::tempdir().unwrap();
+        let kms = KmsService::new(dir.path()).await.unwrap();
+
+        let key = kms.create_key("toggle").await.unwrap();
+        assert!(key.enabled);
+
+        kms.disable_key(&key.key_id).await.unwrap();
+        let k = kms.get_key(&key.key_id).await.unwrap();
+        assert!(!k.enabled);
+
+        kms.enable_key(&key.key_id).await.unwrap();
+        let k = kms.get_key(&key.key_id).await.unwrap();
+        assert!(k.enabled);
+    }
+
+    #[tokio::test]
+    async fn test_delete_key() {
+        let dir = tempfile::tempdir().unwrap();
+        let kms = KmsService::new(dir.path()).await.unwrap();
+
+        let key = kms.create_key("doomed").await.unwrap();
+        assert!(kms.delete_key(&key.key_id).await.unwrap());
+        assert!(kms.get_key(&key.key_id).await.is_none());
+        assert_eq!(kms.list_keys().await.len(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_encrypt_decrypt_data() {
+        let dir = tempfile::tempdir().unwrap();
+        let kms = KmsService::new(dir.path()).await.unwrap();
+
+        let key = kms.create_key("enc-key").await.unwrap();
+        let plaintext = b"Hello, KMS!";
+
+        let ciphertext = kms.encrypt_data(&key.key_id, plaintext).await.unwrap();
+        assert_ne!(&ciphertext, plaintext);
+
+        let decrypted = kms.decrypt_data(&key.key_id, &ciphertext).await.unwrap();
+        assert_eq!(decrypted, plaintext);
+    }
+
+    #[tokio::test]
+    async fn test_generate_data_key() {
+        let dir = tempfile::tempdir().unwrap();
+        let kms = KmsService::new(dir.path()).await.unwrap();
+
+        let key = kms.create_key("data-key-gen").await.unwrap();
+        let (plaintext, wrapped) = kms.generate_data_key(&key.key_id, 32).await.unwrap();
+
+        assert_eq!(plaintext.len(), 32);
+        assert!(wrapped.len() > 32);
+    }
+
+    #[tokio::test]
+    async fn test_disabled_key_cannot_encrypt() {
+        let dir = tempfile::tempdir().unwrap();
+        let kms = KmsService::new(dir.path()).await.unwrap();
+
+        let key = kms.create_key("disabled").await.unwrap();
+        kms.disable_key(&key.key_id).await.unwrap();
+
+        let result = kms.encrypt_data(&key.key_id, b"test").await;
+        assert!(result.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_persistence_across_reload() {
+        let dir = tempfile::tempdir().unwrap();
+
+        let key_id = {
+            let kms = KmsService::new(dir.path()).await.unwrap();
+            let key = kms.create_key("persistent").await.unwrap();
+            key.key_id
+        };
+
+        let kms2 = KmsService::new(dir.path()).await.unwrap();
+        let key = kms2.get_key(&key_id).await;
+        assert!(key.is_some());
+        assert_eq!(key.unwrap().description, "persistent");
+    }
+
+    #[tokio::test]
+    async fn test_master_key_roundtrip() {
+        let dir = tempfile::tempdir().unwrap();
+        let key1 = load_or_create_master_key(dir.path()).await.unwrap();
+        let key2 = load_or_create_master_key(dir.path()).await.unwrap();
+        assert_eq!(key1, key2);
+    }
+}

rust/myfsio-engine/crates/myfsio-crypto/src/lib.rs

@@ -0,0 +1,4 @@
+pub mod hashing;
+pub mod aes_gcm;
+pub mod kms;
+pub mod encryption;