kqjy/MyFSIO
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

First porting of Python to Rust - update docs and bug fixes

Browse Source
This commit is contained in:
kqjy
2026-04-20 21:27:02 +08:00
parent c2ef37b84e
commit 476b9bd2e4
82 changed files with 24682 additions and 4132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
rust/myfsio-engine/crates/myfsio-auth/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "myfsio-auth"
version = "0.1.0"
edition = "2021"
version.workspace = true
edition.workspace = true
[dependencies]
myfsio-common = { path = "../myfsio-common" }
@@ -12,6 +12,7 @@ aes = { workspace = true }
cbc = { workspace = true }
base64 = { workspace = true }
pbkdf2 = "0.12"
rand = "0.8"
lru = { workspace = true }
parking_lot = { workspace = true }
percent-encoding = { workspace = true }

44
rust/myfsio-engine/crates/myfsio-auth/src/fernet.rs
View File

@@ -1,9 +1,11 @@
use aes::cipher::{block_padding::Pkcs7, BlockDecryptMut, KeyIvInit};
use aes::cipher::{block_padding::Pkcs7, BlockDecryptMut, BlockEncryptMut, KeyIvInit};
use base64::{engine::general_purpose::URL_SAFE, Engine};
use hmac::{Hmac, Mac};
use rand::RngCore;
use sha2::Sha256;
type Aes128CbcDec = cbc::Decryptor<aes::Aes128>;
type Aes128CbcEnc = cbc::Encryptor<aes::Aes128>;
type HmacSha256 = Hmac<Sha256>;
pub fn derive_fernet_key(secret: &str) -> String {
@@ -44,8 +46,7 @@ pub fn decrypt(key_b64: &str, token: &str) -> Result<Vec<u8>, &'static str> {
let payload = &token_bytes[..hmac_offset];
let expected_hmac = &token_bytes[hmac_offset..];
let mut mac =
HmacSha256::new_from_slice(signing_key).map_err(|_| "hmac key error")?;
let mut mac = HmacSha256::new_from_slice(signing_key).map_err(|_| "hmac key error")?;
mac.update(payload);
mac.verify_slice(expected_hmac)
.map_err(|_| "HMAC verification failed")?;
@@ -60,6 +61,43 @@ pub fn decrypt(key_b64: &str, token: &str) -> Result<Vec<u8>, &'static str> {
Ok(plaintext)
}
pub fn encrypt(key_b64: &str, plaintext: &[u8]) -> Result<String, &'static str> {
let key_bytes = URL_SAFE
.decode(key_b64)
.map_err(|_| "invalid fernet key base64")?;
if key_bytes.len() != 32 {
return Err("fernet key must be 32 bytes");
}
let signing_key = &key_bytes[..16];
let encryption_key = &key_bytes[16..];
let mut iv = [0u8; 16];
rand::thread_rng().fill_bytes(&mut iv);
let timestamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_err(|_| "system time error")?
.as_secs();
let ciphertext = Aes128CbcEnc::new(encryption_key.into(), (&iv).into())
.encrypt_padded_vec_mut::<Pkcs7>(plaintext);
let mut payload = Vec::with_capacity(1 + 8 + 16 + ciphertext.len());
payload.push(0x80);
payload.extend_from_slice(&timestamp.to_be_bytes());
payload.extend_from_slice(&iv);
payload.extend_from_slice(&ciphertext);
let mut mac = HmacSha256::new_from_slice(signing_key).map_err(|_| "hmac key error")?;
mac.update(&payload);
let tag = mac.finalize().into_bytes();
let mut token_bytes = payload;
token_bytes.extend_from_slice(&tag);
Ok(URL_SAFE.encode(&token_bytes))
}
#[cfg(test)]
mod tests {
use super::*;

321
rust/myfsio-engine/crates/myfsio-auth/src/iam.rs
View File

@@ -69,7 +69,10 @@ impl RawIamUser {
}
}
let display_name = self.display_name.unwrap_or_else(|| {
access_keys.first().map(|k| k.access_key.clone()).unwrap_or_else(|| "unknown".to_string())
access_keys
.first()
.map(|k| k.access_key.clone())
.unwrap_or_else(|| "unknown".to_string())
});
let user_id = self.user_id.unwrap_or_else(|| {
format!("u-{}", display_name.to_ascii_lowercase().replace(' ', "-"))
@@ -173,7 +176,7 @@ impl IamService {
(None, Some(_)) => true,
(Some(old), Some(new)) => old != new,
(Some(_), None) => true,
(None, None) => state.key_secrets.is_empty(),
(None, None) => false,
}
};
@@ -188,7 +191,11 @@ impl IamService {
let content = match std::fs::read_to_string(&self.config_path) {
Ok(c) => c,
Err(e) => {
tracing::warn!("Failed to read IAM config {}: {}", self.config_path.display(), e);
tracing::warn!(
"Failed to read IAM config {}: {}",
self.config_path.display(),
e
);
return;
}
};
@@ -205,7 +212,10 @@ impl IamService {
}
},
Err(e) => {
tracing::error!("Failed to decrypt IAM config: {}. SECRET_KEY may have changed.", e);
tracing::error!(
"Failed to decrypt IAM config: {}. SECRET_KEY may have changed.",
e
);
return;
}
},
@@ -226,7 +236,11 @@ impl IamService {
}
};
let users: Vec<IamUser> = raw_config.users.into_iter().map(|u| u.normalize()).collect();
let users: Vec<IamUser> = raw_config
.users
.into_iter()
.map(|u| u.normalize())
.collect();
let mut key_secrets = HashMap::new();
let mut key_index = HashMap::new();
@@ -254,9 +268,11 @@ impl IamService {
state.file_mtime = file_mtime;
state.last_check = Instant::now();
tracing::info!("IAM config reloaded: {} users, {} keys",
tracing::info!(
"IAM config reloaded: {} users, {} keys",
users.len(),
state.key_secrets.len());
state.key_secrets.len()
);
}
pub fn get_secret_key(&self, access_key: &str) -> Option<String> {
@@ -308,9 +324,10 @@ impl IamService {
}
}
let is_admin = user.policies.iter().any(|p| {
p.bucket == "*" && p.actions.iter().any(|a| a == "*")
});
let is_admin = user
.policies
.iter()
.any(|p| p.bucket == "*" && p.actions.iter().any(|a| a == "*"));
Some(Principal::new(
access_key.to_string(),
@@ -341,10 +358,7 @@ impl IamService {
return true;
}
let normalized_bucket = bucket_name
.unwrap_or("*")
.trim()
.to_ascii_lowercase();
let normalized_bucket = bucket_name.unwrap_or("*").trim().to_ascii_lowercase();
let normalized_action = action.trim().to_ascii_lowercase();
let state = self.state.read();
@@ -383,6 +397,46 @@ impl IamService {
false
}
pub fn export_config(&self, mask_secrets: bool) -> serde_json::Value {
self.reload_if_needed();
let state = self.state.read();
let users: Vec<serde_json::Value> = state
.user_records
.values()
.map(|u| {
let access_keys: Vec<serde_json::Value> = u
.access_keys
.iter()
.map(|k| {
let secret = if mask_secrets {
"***".to_string()
} else {
k.secret_key.clone()
};
serde_json::json!({
"access_key": k.access_key,
"secret_key": secret,
"status": k.status,
"created_at": k.created_at,
})
})
.collect();
serde_json::json!({
"user_id": u.user_id,
"display_name": u.display_name,
"enabled": u.enabled,
"expires_at": u.expires_at,
"access_keys": access_keys,
"policies": u.policies,
})
})
.collect();
serde_json::json!({
"version": 2,
"users": users,
})
}
pub async fn list_users(&self) -> Vec<serde_json::Value> {
self.reload_if_needed();
let state = self.state.read();
@@ -411,12 +465,12 @@ impl IamService {
self.reload_if_needed();
let state = self.state.read();
let user = state
.user_records
.get(identifier)
.or_else(|| {
state.key_index.get(identifier).and_then(|uid| state.user_records.get(uid))
})?;
let user = state.user_records.get(identifier).or_else(|| {
state
.key_index
.get(identifier)
.and_then(|uid| state.user_records.get(uid))
})?;
Some(serde_json::json!({
"user_id": user.user_id,
@@ -449,8 +503,7 @@ impl IamService {
.users
.iter_mut()
.find(|u| {
u.user_id == identifier
|| u.access_keys.iter().any(|k| k.access_key == identifier)
u.user_id == identifier || u.access_keys.iter().any(|k| k.access_key == identifier)
})
.ok_or_else(|| "User not found".to_string())?;
@@ -468,12 +521,12 @@ impl IamService {
pub fn get_user_policies(&self, identifier: &str) -> Option<Vec<serde_json::Value>> {
self.reload_if_needed();
let state = self.state.read();
let user = state
.user_records
.get(identifier)
.or_else(|| {
state.key_index.get(identifier).and_then(|uid| state.user_records.get(uid))
})?;
let user = state.user_records.get(identifier).or_else(|| {
state
.key_index
.get(identifier)
.and_then(|uid| state.user_records.get(uid))
})?;
Some(
user.policies
.iter()
@@ -496,8 +549,7 @@ impl IamService {
.users
.iter_mut()
.find(|u| {
u.user_id == identifier
|| u.access_keys.iter().any(|k| k.access_key == identifier)
u.user_id == identifier || u.access_keys.iter().any(|k| k.access_key == identifier)
})
.ok_or_else(|| format!("User '{}' not found", identifier))?;
@@ -557,6 +609,178 @@ impl IamService {
self.reload();
Ok(())
}
fn load_config(&self) -> Result<IamConfig, String> {
let content = std::fs::read_to_string(&self.config_path)
.map_err(|e| format!("Failed to read IAM config: {}", e))?;
let raw_text = if content.starts_with("MYFSIO_IAM_ENC:") {
let encrypted_token = &content["MYFSIO_IAM_ENC:".len()..];
let key = self.fernet_key.as_ref().ok_or_else(|| {
"IAM config is encrypted but no SECRET_KEY configured".to_string()
})?;
let plaintext = crate::fernet::decrypt(key, encrypted_token.trim())
.map_err(|e| format!("Failed to decrypt IAM config: {}", e))?;
String::from_utf8(plaintext)
.map_err(|e| format!("Decrypted IAM config not UTF-8: {}", e))?
} else {
content
};
let raw: RawIamConfig = serde_json::from_str(&raw_text)
.map_err(|e| format!("Failed to parse IAM config: {}", e))?;
Ok(IamConfig {
version: 2,
users: raw.users.into_iter().map(|u| u.normalize()).collect(),
})
}
fn save_config(&self, config: &IamConfig) -> Result<(), String> {
let json = serde_json::to_string_pretty(config)
.map_err(|e| format!("Failed to serialize IAM config: {}", e))?;
let payload = if let Some(key) = &self.fernet_key {
let token = crate::fernet::encrypt(key, json.as_bytes())
.map_err(|e| format!("Failed to encrypt IAM config: {}", e))?;
format!("MYFSIO_IAM_ENC:{}", token)
} else {
json
};
std::fs::write(&self.config_path, payload)
.map_err(|e| format!("Failed to write IAM config: {}", e))?;
self.reload();
Ok(())
}
pub fn create_user(
&self,
display_name: &str,
policies: Option<Vec<IamPolicy>>,
access_key: Option<String>,
secret_key: Option<String>,
expires_at: Option<String>,
) -> Result<serde_json::Value, String> {
let mut config = self.load_config()?;
let new_ak = access_key
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| format!("AK{}", uuid::Uuid::new_v4().simple()));
let new_sk = secret_key
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| format!("SK{}", uuid::Uuid::new_v4().simple()));
if config
.users
.iter()
.any(|u| u.access_keys.iter().any(|k| k.access_key == new_ak))
{
return Err(format!("Access key '{}' already exists", new_ak));
}
let user_id = format!("u-{}", uuid::Uuid::new_v4().simple());
let resolved_policies = policies.unwrap_or_else(|| {
vec![IamPolicy {
bucket: "*".to_string(),
actions: vec!["*".to_string()],
prefix: "*".to_string(),
}]
});
let user = IamUser {
user_id: user_id.clone(),
display_name: display_name.to_string(),
enabled: true,
expires_at,
access_keys: vec![AccessKey {
access_key: new_ak.clone(),
secret_key: new_sk.clone(),
status: "active".to_string(),
created_at: Some(chrono::Utc::now().to_rfc3339()),
}],
policies: resolved_policies,
};
config.users.push(user);
self.save_config(&config)?;
Ok(serde_json::json!({
"user_id": user_id,
"access_key": new_ak,
"secret_key": new_sk,
"display_name": display_name,
}))
}
pub fn delete_user(&self, identifier: &str) -> Result<(), String> {
let mut config = self.load_config()?;
let before = config.users.len();
config.users.retain(|u| {
u.user_id != identifier && !u.access_keys.iter().any(|k| k.access_key == identifier)
});
if config.users.len() == before {
return Err(format!("User '{}' not found", identifier));
}
self.save_config(&config)
}
pub fn update_user(
&self,
identifier: &str,
display_name: Option<String>,
expires_at: Option<Option<String>>,
) -> Result<(), String> {
let mut config = self.load_config()?;
let user = config
.users
.iter_mut()
.find(|u| {
u.user_id == identifier || u.access_keys.iter().any(|k| k.access_key == identifier)
})
.ok_or_else(|| format!("User '{}' not found", identifier))?;
if let Some(name) = display_name {
user.display_name = name;
}
if let Some(exp) = expires_at {
user.expires_at = exp;
}
self.save_config(&config)
}
pub fn update_user_policies(
&self,
identifier: &str,
policies: Vec<IamPolicy>,
) -> Result<(), String> {
let mut config = self.load_config()?;
let user = config
.users
.iter_mut()
.find(|u| {
u.user_id == identifier || u.access_keys.iter().any(|k| k.access_key == identifier)
})
.ok_or_else(|| format!("User '{}' not found", identifier))?;
user.policies = policies;
self.save_config(&config)
}
pub fn rotate_secret(&self, identifier: &str) -> Result<serde_json::Value, String> {
let mut config = self.load_config()?;
let user = config
.users
.iter_mut()
.find(|u| {
u.user_id == identifier || u.access_keys.iter().any(|k| k.access_key == identifier)
})
.ok_or_else(|| format!("User '{}' not found", identifier))?;
let key = user
.access_keys
.first_mut()
.ok_or_else(|| "User has no access keys".to_string())?;
let new_sk = format!("SK{}", uuid::Uuid::new_v4().simple());
key.secret_key = new_sk.clone();
let ak = key.access_key.clone();
self.save_config(&config)?;
Ok(serde_json::json!({
"access_key": ak,
"secret_key": new_sk,
}))
}
}
fn bucket_matches(policy_bucket: &str, bucket: &str) -> bool {
@@ -622,10 +846,7 @@ mod tests {
let svc = IamService::new(tmp.path().to_path_buf());
let secret = svc.get_secret_key("AKIAIOSFODNN7EXAMPLE");
assert_eq!(
secret.unwrap(),
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
);
assert_eq!(secret.unwrap(), "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
}
#[test]
@@ -664,7 +885,9 @@ mod tests {
tmp.flush().unwrap();
let svc = IamService::new(tmp.path().to_path_buf());
assert!(svc.authenticate("AKIAIOSFODNN7EXAMPLE", "wrongsecret").is_none());
assert!(svc
.authenticate("AKIAIOSFODNN7EXAMPLE", "wrongsecret")
.is_none());
}
#[test]
@@ -784,29 +1007,9 @@ mod tests {
let svc = IamService::new(tmp.path().to_path_buf());
let principal = svc.get_principal("READER_KEY").unwrap();
assert!(svc.authorize(
&principal,
Some("docs"),
"read",
Some("reports/2026.csv"),
));
assert!(!svc.authorize(
&principal,
Some("docs"),
"write",
Some("reports/2026.csv"),
));
assert!(!svc.authorize(
&principal,
Some("docs"),
"read",
Some("private/2026.csv"),
));
assert!(!svc.authorize(
&principal,
Some("other"),
"read",
Some("reports/2026.csv"),
));
assert!(svc.authorize(&principal, Some("docs"), "read", Some("reports/2026.csv"),));
assert!(!svc.authorize(&principal, Some("docs"), "write", Some("reports/2026.csv"),));
assert!(!svc.authorize(&principal, Some("docs"), "read", Some("private/2026.csv"),));
assert!(!svc.authorize(&principal, Some("other"), "read", Some("reports/2026.csv"),));
}
}

6
rust/myfsio-engine/crates/myfsio-auth/src/lib.rs
View File

@@ -1,4 +1,4 @@
pub mod sigv4;
pub mod principal;
pub mod iam;
mod fernet;
pub mod iam;
pub mod principal;
pub mod sigv4;

38
rust/myfsio-engine/crates/myfsio-auth/src/sigv4.rs
View File

@@ -64,7 +64,10 @@ pub fn derive_signing_key_cached(
}
}
let k_date = hmac_sha256(format!("AWS4{}", secret_key).as_bytes(), date_stamp.as_bytes());
let k_date = hmac_sha256(
format!("AWS4{}", secret_key).as_bytes(),
date_stamp.as_bytes(),
);
let k_region = hmac_sha256(&k_date, region.as_bytes());
let k_service = hmac_sha256(&k_region, service.as_bytes());
let k_signing = hmac_sha256(&k_service, b"aws4_request");
@@ -134,7 +137,11 @@ pub fn verify_sigv4_signature(
let canonical_request = format!(
"{}\n{}\n{}\n{}\n{}\n{}",
method, canonical_uri, canonical_query_string, canonical_headers, signed_headers_str,
method,
canonical_uri,
canonical_query_string,
canonical_headers,
signed_headers_str,
payload_hash
);
@@ -197,7 +204,12 @@ mod tests {
#[test]
fn test_derive_signing_key() {
let key = derive_signing_key("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "20130524", "us-east-1", "s3");
let key = derive_signing_key(
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
"20130524",
"us-east-1",
"s3",
);
assert_eq!(key.len(), 32);
}
@@ -217,7 +229,11 @@ mod tests {
#[test]
fn test_build_string_to_sign() {
let result = build_string_to_sign("20130524T000000Z", "20130524/us-east-1/s3/aws4_request", "GET\n/\n\nhost:example.com\n\nhost\nUNSIGNED-PAYLOAD");
let result = build_string_to_sign(
"20130524T000000Z",
"20130524/us-east-1/s3/aws4_request",
"GET\n/\n\nhost:example.com\n\nhost\nUNSIGNED-PAYLOAD",
);
assert!(result.starts_with("AWS4-HMAC-SHA256\n"));
assert!(result.contains("20130524T000000Z"));
}
@@ -239,8 +255,13 @@ mod tests {
let signing_key = derive_signing_key(secret, date_stamp, region, service);
let canonical_request = "GET\n/\n\nhost:examplebucket.s3.amazonaws.com\n\nhost\nUNSIGNED-PAYLOAD";
let string_to_sign = build_string_to_sign(amz_date, &format!("{}/{}/{}/aws4_request", date_stamp, region, service), canonical_request);
let canonical_request =
"GET\n/\n\nhost:examplebucket.s3.amazonaws.com\n\nhost\nUNSIGNED-PAYLOAD";
let string_to_sign = build_string_to_sign(
amz_date,
&format!("{}/{}/{}/aws4_request", date_stamp, region, service),
canonical_request,
);
let signature = compute_signature(&signing_key, &string_to_sign);
@@ -249,7 +270,10 @@ mod tests {
"/",
&[],
"host",
&[("host".to_string(), "examplebucket.s3.amazonaws.com".to_string())],
&[(
"host".to_string(),
"examplebucket.s3.amazonaws.com".to_string(),
)],
"UNSIGNED-PAYLOAD",
amz_date,
date_stamp,