kqjy/MyFSIO
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

First porting of Python to Rust - update docs and bug fixes

Browse Source
This commit is contained in:
kqjy
2026-04-20 21:27:02 +08:00
parent c2ef37b84e
commit 476b9bd2e4
82 changed files with 24682 additions and 4132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

178
rust/myfsio-engine/Cargo.lock generated
View File

@@ -326,6 +326,18 @@ dependencies = [
"regex-syntax",
]
[[package]]
name = "async-compression"
version = "0.4.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1"
dependencies = [
"compression-codecs",
"compression-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "async-trait"
version = "0.1.89"
@@ -1165,6 +1177,23 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "compression-codecs"
version = "0.4.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7"
dependencies = [
"compression-core",
"flate2",
"memchr",
]
[[package]]
name = "compression-core"
version = "0.4.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d"
[[package]]
name = "const-oid"
version = "0.9.6"
@@ -1442,6 +1471,12 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "dotenvy"
version = "0.15.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
[[package]]
name = "duckdb"
version = "1.10501.0"
@@ -2137,7 +2172,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
"windows-core",
"windows-core 0.62.2",
]
[[package]]
@@ -2595,7 +2630,7 @@ dependencies = [
[[package]]
name = "myfsio-auth"
version = "0.1.0"
version = "0.5.0"
dependencies = [
"aes",
"base64",
@@ -2608,6 +2643,7 @@ dependencies = [
"parking_lot",
"pbkdf2",
"percent-encoding",
"rand 0.8.5",
"serde",
"serde_json",
"sha2 0.10.9",
@@ -2619,7 +2655,7 @@ dependencies = [
[[package]]
name = "myfsio-common"
version = "0.1.0"
version = "0.5.0"
dependencies = [
"chrono",
"serde",
@@ -2630,7 +2666,7 @@ dependencies = [
[[package]]
name = "myfsio-crypto"
version = "0.1.0"
version = "0.5.0"
dependencies = [
"aes-gcm",
"base64",
@@ -2651,8 +2687,9 @@ dependencies = [
[[package]]
name = "myfsio-server"
version = "0.1.0"
version = "0.5.0"
dependencies = [
"aes-gcm",
"async-trait",
"aws-config",
"aws-credential-types",
@@ -2665,6 +2702,7 @@ dependencies = [
"clap",
"cookie",
"crc32fast",
"dotenvy",
"duckdb",
"futures",
"http-body-util",
@@ -2686,6 +2724,7 @@ dependencies = [
"serde",
"serde_json",
"subtle",
"sysinfo",
"tempfile",
"tera",
"tokio",
@@ -2699,7 +2738,7 @@ dependencies = [
[[package]]
name = "myfsio-storage"
version = "0.1.0"
version = "0.5.0"
dependencies = [
"chrono",
"dashmap",
@@ -2722,7 +2761,7 @@ dependencies = [
[[package]]
name = "myfsio-xml"
version = "0.1.0"
version = "0.5.0"
dependencies = [
"chrono",
"myfsio-common",
@@ -2730,6 +2769,15 @@ dependencies = [
"serde",
]
[[package]]
name = "ntapi"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae"
dependencies = [
"winapi",
]
[[package]]
name = "nu-ansi-term"
version = "0.50.3"
@@ -3222,6 +3270,26 @@ dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "rayon"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
@@ -3905,6 +3973,20 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "sysinfo"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c33cd241af0f2e9e3b5c32163b873b29956890b5342e6745b917ce9d490f4af"
dependencies = [
"core-foundation-sys",
"libc",
"memchr",
"ntapi",
"rayon",
"windows",
]
[[package]]
name = "tap"
version = "1.0.1"
@@ -4164,6 +4246,7 @@ version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"
dependencies = [
"async-compression",
"bitflags",
"bytes",
"futures-core",
@@ -4569,6 +4652,22 @@ dependencies = [
"rustls-pki-types",
]
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.11"
@@ -4578,19 +4677,58 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"
dependencies = [
"windows-core 0.57.0",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-core"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d"
dependencies = [
"windows-implement 0.57.0",
"windows-interface 0.57.0",
"windows-result 0.1.2",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-core"
version = "0.62.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
dependencies = [
"windows-implement",
"windows-interface",
"windows-implement 0.60.2",
"windows-interface 0.59.3",
"windows-link",
"windows-result",
"windows-result 0.4.1",
"windows-strings",
]
[[package]]
name = "windows-implement"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "windows-implement"
version = "0.60.2"
@@ -4602,6 +4740,17 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "windows-interface"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "windows-interface"
version = "0.59.3"
@@ -4619,6 +4768,15 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-result"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-result"
version = "0.4.1"

7
rust/myfsio-engine/Cargo.toml
View File

@@ -9,11 +9,15 @@ members = [
"crates/myfsio-server",
]
[workspace.package]
version = "0.4.3"
edition = "2021"
[workspace.dependencies]
tokio = { version = "1", features = ["full"] }
axum = { version = "0.8" }
tower = { version = "0.5" }
tower-http = { version = "0.6", features = ["cors", "trace", "fs"] }
tower-http = { version = "0.6", features = ["cors", "trace", "fs", "compression-gzip"] }
hyper = { version = "1" }
bytes = "1"
serde = { version = "1", features = ["derive"] }
@@ -54,3 +58,4 @@ tera = "1"
cookie = "0.18"
subtle = "2"
clap = { version = "4", features = ["derive"] }
dotenvy = "0.15"

5
rust/myfsio-engine/crates/myfsio-auth/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "myfsio-auth"
version = "0.1.0"
edition = "2021"
version.workspace = true
edition.workspace = true
[dependencies]
myfsio-common = { path = "../myfsio-common" }
@@ -12,6 +12,7 @@ aes = { workspace = true }
cbc = { workspace = true }
base64 = { workspace = true }
pbkdf2 = "0.12"
rand = "0.8"
lru = { workspace = true }
parking_lot = { workspace = true }
percent-encoding = { workspace = true }

44
rust/myfsio-engine/crates/myfsio-auth/src/fernet.rs
View File

@@ -1,9 +1,11 @@
use aes::cipher::{block_padding::Pkcs7, BlockDecryptMut, KeyIvInit};
use aes::cipher::{block_padding::Pkcs7, BlockDecryptMut, BlockEncryptMut, KeyIvInit};
use base64::{engine::general_purpose::URL_SAFE, Engine};
use hmac::{Hmac, Mac};
use rand::RngCore;
use sha2::Sha256;
type Aes128CbcDec = cbc::Decryptor<aes::Aes128>;
type Aes128CbcEnc = cbc::Encryptor<aes::Aes128>;
type HmacSha256 = Hmac<Sha256>;
pub fn derive_fernet_key(secret: &str) -> String {
@@ -44,8 +46,7 @@ pub fn decrypt(key_b64: &str, token: &str) -> Result<Vec<u8>, &'static str> {
let payload = &token_bytes[..hmac_offset];
let expected_hmac = &token_bytes[hmac_offset..];
let mut mac =
HmacSha256::new_from_slice(signing_key).map_err(|_| "hmac key error")?;
let mut mac = HmacSha256::new_from_slice(signing_key).map_err(|_| "hmac key error")?;
mac.update(payload);
mac.verify_slice(expected_hmac)
.map_err(|_| "HMAC verification failed")?;
@@ -60,6 +61,43 @@ pub fn decrypt(key_b64: &str, token: &str) -> Result<Vec<u8>, &'static str> {
Ok(plaintext)
}
pub fn encrypt(key_b64: &str, plaintext: &[u8]) -> Result<String, &'static str> {
let key_bytes = URL_SAFE
.decode(key_b64)
.map_err(|_| "invalid fernet key base64")?;
if key_bytes.len() != 32 {
return Err("fernet key must be 32 bytes");
}
let signing_key = &key_bytes[..16];
let encryption_key = &key_bytes[16..];
let mut iv = [0u8; 16];
rand::thread_rng().fill_bytes(&mut iv);
let timestamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_err(|_| "system time error")?
.as_secs();
let ciphertext = Aes128CbcEnc::new(encryption_key.into(), (&iv).into())
.encrypt_padded_vec_mut::<Pkcs7>(plaintext);
let mut payload = Vec::with_capacity(1 + 8 + 16 + ciphertext.len());
payload.push(0x80);
payload.extend_from_slice(&timestamp.to_be_bytes());
payload.extend_from_slice(&iv);
payload.extend_from_slice(&ciphertext);
let mut mac = HmacSha256::new_from_slice(signing_key).map_err(|_| "hmac key error")?;
mac.update(&payload);
let tag = mac.finalize().into_bytes();
let mut token_bytes = payload;
token_bytes.extend_from_slice(&tag);
Ok(URL_SAFE.encode(&token_bytes))
}
#[cfg(test)]
mod tests {
use super::*;

321
rust/myfsio-engine/crates/myfsio-auth/src/iam.rs
View File

@@ -69,7 +69,10 @@ impl RawIamUser {
}
}
let display_name = self.display_name.unwrap_or_else(|| {
access_keys.first().map(|k| k.access_key.clone()).unwrap_or_else(|| "unknown".to_string())
access_keys
.first()
.map(|k| k.access_key.clone())
.unwrap_or_else(|| "unknown".to_string())
});
let user_id = self.user_id.unwrap_or_else(|| {
format!("u-{}", display_name.to_ascii_lowercase().replace(' ', "-"))
@@ -173,7 +176,7 @@ impl IamService {
(None, Some(_)) => true,
(Some(old), Some(new)) => old != new,
(Some(_), None) => true,
(None, None) => state.key_secrets.is_empty(),
(None, None) => false,
}
};
@@ -188,7 +191,11 @@ impl IamService {
let content = match std::fs::read_to_string(&self.config_path) {
Ok(c) => c,
Err(e) => {
tracing::warn!("Failed to read IAM config {}: {}", self.config_path.display(), e);
tracing::warn!(
"Failed to read IAM config {}: {}",
self.config_path.display(),
e
);
return;
}
};
@@ -205,7 +212,10 @@ impl IamService {
}
},
Err(e) => {
tracing::error!("Failed to decrypt IAM config: {}. SECRET_KEY may have changed.", e);
tracing::error!(
"Failed to decrypt IAM config: {}. SECRET_KEY may have changed.",
e
);
return;
}
},
@@ -226,7 +236,11 @@ impl IamService {
}
};
let users: Vec<IamUser> = raw_config.users.into_iter().map(|u| u.normalize()).collect();
let users: Vec<IamUser> = raw_config
.users
.into_iter()
.map(|u| u.normalize())
.collect();
let mut key_secrets = HashMap::new();
let mut key_index = HashMap::new();
@@ -254,9 +268,11 @@ impl IamService {
state.file_mtime = file_mtime;
state.last_check = Instant::now();
tracing::info!("IAM config reloaded: {} users, {} keys",
tracing::info!(
"IAM config reloaded: {} users, {} keys",
users.len(),
state.key_secrets.len());
state.key_secrets.len()
);
}
pub fn get_secret_key(&self, access_key: &str) -> Option<String> {
@@ -308,9 +324,10 @@ impl IamService {
}
}
let is_admin = user.policies.iter().any(|p| {
p.bucket == "*" && p.actions.iter().any(|a| a == "*")
});
let is_admin = user
.policies
.iter()
.any(|p| p.bucket == "*" && p.actions.iter().any(|a| a == "*"));
Some(Principal::new(
access_key.to_string(),
@@ -341,10 +358,7 @@ impl IamService {
return true;
}
let normalized_bucket = bucket_name
.unwrap_or("*")
.trim()
.to_ascii_lowercase();
let normalized_bucket = bucket_name.unwrap_or("*").trim().to_ascii_lowercase();
let normalized_action = action.trim().to_ascii_lowercase();
let state = self.state.read();
@@ -383,6 +397,46 @@ impl IamService {
false
}
pub fn export_config(&self, mask_secrets: bool) -> serde_json::Value {
self.reload_if_needed();
let state = self.state.read();
let users: Vec<serde_json::Value> = state
.user_records
.values()
.map(|u| {
let access_keys: Vec<serde_json::Value> = u
.access_keys
.iter()
.map(|k| {
let secret = if mask_secrets {
"***".to_string()
} else {
k.secret_key.clone()
};
serde_json::json!({
"access_key": k.access_key,
"secret_key": secret,
"status": k.status,
"created_at": k.created_at,
})
})
.collect();
serde_json::json!({
"user_id": u.user_id,
"display_name": u.display_name,
"enabled": u.enabled,
"expires_at": u.expires_at,
"access_keys": access_keys,
"policies": u.policies,
})
})
.collect();
serde_json::json!({
"version": 2,
"users": users,
})
}
pub async fn list_users(&self) -> Vec<serde_json::Value> {
self.reload_if_needed();
let state = self.state.read();
@@ -411,12 +465,12 @@ impl IamService {
self.reload_if_needed();
let state = self.state.read();
let user = state
.user_records
.get(identifier)
.or_else(|| {
state.key_index.get(identifier).and_then(|uid| state.user_records.get(uid))
})?;
let user = state.user_records.get(identifier).or_else(|| {
state
.key_index
.get(identifier)
.and_then(|uid| state.user_records.get(uid))
})?;
Some(serde_json::json!({
"user_id": user.user_id,
@@ -449,8 +503,7 @@ impl IamService {
.users
.iter_mut()
.find(|u| {
u.user_id == identifier
|| u.access_keys.iter().any(|k| k.access_key == identifier)
u.user_id == identifier || u.access_keys.iter().any(|k| k.access_key == identifier)
})
.ok_or_else(|| "User not found".to_string())?;
@@ -468,12 +521,12 @@ impl IamService {
pub fn get_user_policies(&self, identifier: &str) -> Option<Vec<serde_json::Value>> {
self.reload_if_needed();
let state = self.state.read();
let user = state
.user_records
.get(identifier)
.or_else(|| {
state.key_index.get(identifier).and_then(|uid| state.user_records.get(uid))
})?;
let user = state.user_records.get(identifier).or_else(|| {
state
.key_index
.get(identifier)
.and_then(|uid| state.user_records.get(uid))
})?;
Some(
user.policies
.iter()
@@ -496,8 +549,7 @@ impl IamService {
.users
.iter_mut()
.find(|u| {
u.user_id == identifier
|| u.access_keys.iter().any(|k| k.access_key == identifier)
u.user_id == identifier || u.access_keys.iter().any(|k| k.access_key == identifier)
})
.ok_or_else(|| format!("User '{}' not found", identifier))?;
@@ -557,6 +609,178 @@ impl IamService {
self.reload();
Ok(())
}
fn load_config(&self) -> Result<IamConfig, String> {
let content = std::fs::read_to_string(&self.config_path)
.map_err(|e| format!("Failed to read IAM config: {}", e))?;
let raw_text = if content.starts_with("MYFSIO_IAM_ENC:") {
let encrypted_token = &content["MYFSIO_IAM_ENC:".len()..];
let key = self.fernet_key.as_ref().ok_or_else(|| {
"IAM config is encrypted but no SECRET_KEY configured".to_string()
})?;
let plaintext = crate::fernet::decrypt(key, encrypted_token.trim())
.map_err(|e| format!("Failed to decrypt IAM config: {}", e))?;
String::from_utf8(plaintext)
.map_err(|e| format!("Decrypted IAM config not UTF-8: {}", e))?
} else {
content
};
let raw: RawIamConfig = serde_json::from_str(&raw_text)
.map_err(|e| format!("Failed to parse IAM config: {}", e))?;
Ok(IamConfig {
version: 2,
users: raw.users.into_iter().map(|u| u.normalize()).collect(),
})
}
fn save_config(&self, config: &IamConfig) -> Result<(), String> {
let json = serde_json::to_string_pretty(config)
.map_err(|e| format!("Failed to serialize IAM config: {}", e))?;
let payload = if let Some(key) = &self.fernet_key {
let token = crate::fernet::encrypt(key, json.as_bytes())
.map_err(|e| format!("Failed to encrypt IAM config: {}", e))?;
format!("MYFSIO_IAM_ENC:{}", token)
} else {
json
};
std::fs::write(&self.config_path, payload)
.map_err(|e| format!("Failed to write IAM config: {}", e))?;
self.reload();
Ok(())
}
pub fn create_user(
&self,
display_name: &str,
policies: Option<Vec<IamPolicy>>,
access_key: Option<String>,
secret_key: Option<String>,
expires_at: Option<String>,
) -> Result<serde_json::Value, String> {
let mut config = self.load_config()?;
let new_ak = access_key
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| format!("AK{}", uuid::Uuid::new_v4().simple()));
let new_sk = secret_key
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| format!("SK{}", uuid::Uuid::new_v4().simple()));
if config
.users
.iter()
.any(|u| u.access_keys.iter().any(|k| k.access_key == new_ak))
{
return Err(format!("Access key '{}' already exists", new_ak));
}
let user_id = format!("u-{}", uuid::Uuid::new_v4().simple());
let resolved_policies = policies.unwrap_or_else(|| {
vec![IamPolicy {
bucket: "*".to_string(),
actions: vec!["*".to_string()],
prefix: "*".to_string(),
}]
});
let user = IamUser {
user_id: user_id.clone(),
display_name: display_name.to_string(),
enabled: true,
expires_at,
access_keys: vec![AccessKey {
access_key: new_ak.clone(),
secret_key: new_sk.clone(),
status: "active".to_string(),
created_at: Some(chrono::Utc::now().to_rfc3339()),
}],
policies: resolved_policies,
};
config.users.push(user);
self.save_config(&config)?;
Ok(serde_json::json!({
"user_id": user_id,
"access_key": new_ak,
"secret_key": new_sk,
"display_name": display_name,
}))
}
pub fn delete_user(&self, identifier: &str) -> Result<(), String> {
let mut config = self.load_config()?;
let before = config.users.len();
config.users.retain(|u| {
u.user_id != identifier && !u.access_keys.iter().any(|k| k.access_key == identifier)
});
if config.users.len() == before {
return Err(format!("User '{}' not found", identifier));
}
self.save_config(&config)
}
pub fn update_user(
&self,
identifier: &str,
display_name: Option<String>,
expires_at: Option<Option<String>>,
) -> Result<(), String> {
let mut config = self.load_config()?;
let user = config
.users
.iter_mut()
.find(|u| {
u.user_id == identifier || u.access_keys.iter().any(|k| k.access_key == identifier)
})
.ok_or_else(|| format!("User '{}' not found", identifier))?;
if let Some(name) = display_name {
user.display_name = name;
}
if let Some(exp) = expires_at {
user.expires_at = exp;
}
self.save_config(&config)
}
pub fn update_user_policies(
&self,
identifier: &str,
policies: Vec<IamPolicy>,
) -> Result<(), String> {
let mut config = self.load_config()?;
let user = config
.users
.iter_mut()
.find(|u| {
u.user_id == identifier || u.access_keys.iter().any(|k| k.access_key == identifier)
})
.ok_or_else(|| format!("User '{}' not found", identifier))?;
user.policies = policies;
self.save_config(&config)
}
pub fn rotate_secret(&self, identifier: &str) -> Result<serde_json::Value, String> {
let mut config = self.load_config()?;
let user = config
.users
.iter_mut()
.find(|u| {
u.user_id == identifier || u.access_keys.iter().any(|k| k.access_key == identifier)
})
.ok_or_else(|| format!("User '{}' not found", identifier))?;
let key = user
.access_keys
.first_mut()
.ok_or_else(|| "User has no access keys".to_string())?;
let new_sk = format!("SK{}", uuid::Uuid::new_v4().simple());
key.secret_key = new_sk.clone();
let ak = key.access_key.clone();
self.save_config(&config)?;
Ok(serde_json::json!({
"access_key": ak,
"secret_key": new_sk,
}))
}
}
fn bucket_matches(policy_bucket: &str, bucket: &str) -> bool {
@@ -622,10 +846,7 @@ mod tests {
let svc = IamService::new(tmp.path().to_path_buf());
let secret = svc.get_secret_key("AKIAIOSFODNN7EXAMPLE");
assert_eq!(
secret.unwrap(),
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
);
assert_eq!(secret.unwrap(), "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
}
#[test]
@@ -664,7 +885,9 @@ mod tests {
tmp.flush().unwrap();
let svc = IamService::new(tmp.path().to_path_buf());
assert!(svc.authenticate("AKIAIOSFODNN7EXAMPLE", "wrongsecret").is_none());
assert!(svc
.authenticate("AKIAIOSFODNN7EXAMPLE", "wrongsecret")
.is_none());
}
#[test]
@@ -784,29 +1007,9 @@ mod tests {
let svc = IamService::new(tmp.path().to_path_buf());
let principal = svc.get_principal("READER_KEY").unwrap();
assert!(svc.authorize(
&principal,
Some("docs"),
"read",
Some("reports/2026.csv"),
));
assert!(!svc.authorize(
&principal,
Some("docs"),
"write",
Some("reports/2026.csv"),
));
assert!(!svc.authorize(
&principal,
Some("docs"),
"read",
Some("private/2026.csv"),
));
assert!(!svc.authorize(
&principal,
Some("other"),
"read",
Some("reports/2026.csv"),
));
assert!(svc.authorize(&principal, Some("docs"), "read", Some("reports/2026.csv"),));
assert!(!svc.authorize(&principal, Some("docs"), "write", Some("reports/2026.csv"),));
assert!(!svc.authorize(&principal, Some("docs"), "read", Some("private/2026.csv"),));
assert!(!svc.authorize(&principal, Some("other"), "read", Some("reports/2026.csv"),));
}
}

6
rust/myfsio-engine/crates/myfsio-auth/src/lib.rs
View File

@@ -1,4 +1,4 @@
pub mod sigv4;
pub mod principal;
pub mod iam;
mod fernet;
pub mod iam;
pub mod principal;
pub mod sigv4;

38
rust/myfsio-engine/crates/myfsio-auth/src/sigv4.rs
View File

@@ -64,7 +64,10 @@ pub fn derive_signing_key_cached(
}
}
let k_date = hmac_sha256(format!("AWS4{}", secret_key).as_bytes(), date_stamp.as_bytes());
let k_date = hmac_sha256(
format!("AWS4{}", secret_key).as_bytes(),
date_stamp.as_bytes(),
);
let k_region = hmac_sha256(&k_date, region.as_bytes());
let k_service = hmac_sha256(&k_region, service.as_bytes());
let k_signing = hmac_sha256(&k_service, b"aws4_request");
@@ -134,7 +137,11 @@ pub fn verify_sigv4_signature(
let canonical_request = format!(
"{}\n{}\n{}\n{}\n{}\n{}",
method, canonical_uri, canonical_query_string, canonical_headers, signed_headers_str,
method,
canonical_uri,
canonical_query_string,
canonical_headers,
signed_headers_str,
payload_hash
);
@@ -197,7 +204,12 @@ mod tests {
#[test]
fn test_derive_signing_key() {
let key = derive_signing_key("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "20130524", "us-east-1", "s3");
let key = derive_signing_key(
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
"20130524",
"us-east-1",
"s3",
);
assert_eq!(key.len(), 32);
}
@@ -217,7 +229,11 @@ mod tests {
#[test]
fn test_build_string_to_sign() {
let result = build_string_to_sign("20130524T000000Z", "20130524/us-east-1/s3/aws4_request", "GET\n/\n\nhost:example.com\n\nhost\nUNSIGNED-PAYLOAD");
let result = build_string_to_sign(
"20130524T000000Z",
"20130524/us-east-1/s3/aws4_request",
"GET\n/\n\nhost:example.com\n\nhost\nUNSIGNED-PAYLOAD",
);
assert!(result.starts_with("AWS4-HMAC-SHA256\n"));
assert!(result.contains("20130524T000000Z"));
}
@@ -239,8 +255,13 @@ mod tests {
let signing_key = derive_signing_key(secret, date_stamp, region, service);
let canonical_request = "GET\n/\n\nhost:examplebucket.s3.amazonaws.com\n\nhost\nUNSIGNED-PAYLOAD";
let string_to_sign = build_string_to_sign(amz_date, &format!("{}/{}/{}/aws4_request", date_stamp, region, service), canonical_request);
let canonical_request =
"GET\n/\n\nhost:examplebucket.s3.amazonaws.com\n\nhost\nUNSIGNED-PAYLOAD";
let string_to_sign = build_string_to_sign(
amz_date,
&format!("{}/{}/{}/aws4_request", date_stamp, region, service),
canonical_request,
);
let signature = compute_signature(&signing_key, &string_to_sign);
@@ -249,7 +270,10 @@ mod tests {
"/",
&[],
"host",
&[("host".to_string(), "examplebucket.s3.amazonaws.com".to_string())],
&[(
"host".to_string(),
"examplebucket.s3.amazonaws.com".to_string(),
)],
"UNSIGNED-PAYLOAD",
amz_date,
date_stamp,

4
rust/myfsio-engine/crates/myfsio-common/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "myfsio-common"
version = "0.1.0"
edition = "2021"
version.workspace = true
edition.workspace = true
[dependencies]
thiserror = { workspace = true }

4
rust/myfsio-engine/crates/myfsio-crypto/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "myfsio-crypto"
version = "0.1.0"
edition = "2021"
version.workspace = true
edition.workspace = true
[dependencies]
myfsio-common = { path = "../myfsio-common" }

23
rust/myfsio-engine/crates/myfsio-crypto/src/aes_gcm.rs
View File

@@ -193,7 +193,10 @@ mod tests {
let decrypted = dir.path().join("decrypted.bin");
let data = b"Hello, this is a test of AES-256-GCM chunked encryption!";
std::fs::File::create(&input).unwrap().write_all(data).unwrap();
std::fs::File::create(&input)
.unwrap()
.write_all(data)
.unwrap();
let key = [0x42u8; 32];
let nonce = [0x01u8; 12];
@@ -212,9 +215,18 @@ mod tests {
fn test_invalid_key_size() {
let dir = tempfile::tempdir().unwrap();
let input = dir.path().join("input.bin");
std::fs::File::create(&input).unwrap().write_all(b"test").unwrap();
std::fs::File::create(&input)
.unwrap()
.write_all(b"test")
.unwrap();
let result = encrypt_stream_chunked(&input, &dir.path().join("out"), &[0u8; 16], &[0u8; 12], None);
let result = encrypt_stream_chunked(
&input,
&dir.path().join("out"),
&[0u8; 16],
&[0u8; 12],
None,
);
assert!(matches!(result, Err(CryptoError::InvalidKeySize(16))));
}
@@ -225,7 +237,10 @@ mod tests {
let encrypted = dir.path().join("encrypted.bin");
let decrypted = dir.path().join("decrypted.bin");
std::fs::File::create(&input).unwrap().write_all(b"secret data").unwrap();
std::fs::File::create(&input)
.unwrap()
.write_all(b"secret data")
.unwrap();
let key = [0x42u8; 32];
let nonce = [0x01u8; 12];

52
rust/myfsio-engine/crates/myfsio-crypto/src/encryption.rs
View File

@@ -4,9 +4,7 @@ use rand::RngCore;
use std::collections::HashMap;
use std::path::Path;
use crate::aes_gcm::{
encrypt_stream_chunked, decrypt_stream_chunked, CryptoError,
};
use crate::aes_gcm::{decrypt_stream_chunked, encrypt_stream_chunked, CryptoError};
use crate::kms::KmsService;
#[derive(Debug, Clone, PartialEq)]
@@ -172,15 +170,14 @@ impl EncryptionService {
let ciphertext = kms.encrypt_data(kid, &data_key).await?;
(Some(B64.encode(&ciphertext)), Some(kid.clone()))
}
SseAlgorithm::CustomerProvided => {
(None, None)
}
SseAlgorithm::CustomerProvided => (None, None),
};
let actual_key = if ctx.algorithm == SseAlgorithm::CustomerProvided {
let ck = ctx.customer_key.as_ref().ok_or_else(|| {
CryptoError::EncryptionFailed("No customer key provided".into())
})?;
let ck = ctx
.customer_key
.as_ref()
.ok_or_else(|| CryptoError::EncryptionFailed("No customer key provided".into()))?;
if ck.len() != 32 {
return Err(CryptoError::InvalidKeySize(ck.len()));
}
@@ -195,11 +192,9 @@ impl EncryptionService {
let op = output_path.to_owned();
let ak = actual_key;
let n = nonce;
tokio::task::spawn_blocking(move || {
encrypt_stream_chunked(&ip, &op, &ak, &n, None)
})
.await
.map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))??;
tokio::task::spawn_blocking(move || encrypt_stream_chunked(&ip, &op, &ak, &n, None))
.await
.map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))??;
Ok(EncryptionMetadata {
algorithm: ctx.algorithm.as_str().to_string(),
@@ -216,9 +211,9 @@ impl EncryptionService {
enc_meta: &EncryptionMetadata,
customer_key: Option<&[u8]>,
) -> Result<(), CryptoError> {
let nonce_bytes = B64.decode(&enc_meta.nonce).map_err(|e| {
CryptoError::EncryptionFailed(format!("Bad nonce encoding: {}", e))
})?;
let nonce_bytes = B64
.decode(&enc_meta.nonce)
.map_err(|e| CryptoError::EncryptionFailed(format!("Bad nonce encoding: {}", e)))?;
if nonce_bytes.len() != 12 {
return Err(CryptoError::InvalidNonceSize(nonce_bytes.len()));
}
@@ -262,11 +257,9 @@ impl EncryptionService {
let ip = input_path.to_owned();
let op = output_path.to_owned();
let nb: [u8; 12] = nonce_bytes.try_into().unwrap();
tokio::task::spawn_blocking(move || {
decrypt_stream_chunked(&ip, &op, &data_key, &nb)
})
.await
.map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))??;
tokio::task::spawn_blocking(move || decrypt_stream_chunked(&ip, &op, &data_key, &nb))
.await
.map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))??;
Ok(())
}
@@ -298,7 +291,10 @@ mod tests {
let decrypted = dir.path().join("dec.bin");
let data = b"SSE-S3 encrypted content for testing!";
std::fs::File::create(&input).unwrap().write_all(data).unwrap();
std::fs::File::create(&input)
.unwrap()
.write_all(data)
.unwrap();
let svc = EncryptionService::new(test_master_key(), None);
@@ -328,7 +324,10 @@ mod tests {
let decrypted = dir.path().join("dec.bin");
let data = b"SSE-C encrypted content!";
std::fs::File::create(&input).unwrap().write_all(data).unwrap();
std::fs::File::create(&input)
.unwrap()
.write_all(data)
.unwrap();
let customer_key = [0xBBu8; 32];
let svc = EncryptionService::new(test_master_key(), None);
@@ -369,7 +368,10 @@ mod tests {
fn test_is_encrypted() {
let mut meta = HashMap::new();
assert!(!EncryptionMetadata::is_encrypted(&meta));
meta.insert("x-amz-server-side-encryption".to_string(), "AES256".to_string());
meta.insert(
"x-amz-server-side-encryption".to_string(),
"AES256".to_string(),
);
assert!(EncryptionMetadata::is_encrypted(&meta));
}
}

10
rust/myfsio-engine/crates/myfsio-crypto/src/hashing.rs
View File

@@ -99,7 +99,10 @@ mod tests {
#[test]
fn test_sha256_bytes() {
let hash = sha256_bytes(b"hello");
assert_eq!(hash, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824");
assert_eq!(
hash,
"2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"
);
}
#[test]
@@ -118,7 +121,10 @@ mod tests {
tmp.flush().unwrap();
let (md5, sha) = md5_sha256_file(tmp.path()).unwrap();
assert_eq!(md5, "5d41402abc4b2a76b9719d911017c592");
assert_eq!(sha, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824");
assert_eq!(
sha,
"2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"
);
}
#[tokio::test]

4
rust/myfsio-engine/crates/myfsio-crypto/src/kms.rs
View File

@@ -132,9 +132,7 @@ impl KmsService {
async fn save(&self) -> Result<(), CryptoError> {
let keys = self.keys.read().await;
let store = KmsStore {
keys: keys.clone(),
};
let store = KmsStore { keys: keys.clone() };
let json = serde_json::to_string_pretty(&store)
.map_err(|e| CryptoError::EncryptionFailed(e.to_string()))?;
std::fs::write(&self.keys_path, json).map_err(CryptoError::Io)?;

4
rust/myfsio-engine/crates/myfsio-crypto/src/lib.rs
View File

@@ -1,4 +1,4 @@
pub mod hashing;
pub mod aes_gcm;
pub mod kms;
pub mod encryption;
pub mod hashing;
pub mod kms;

7
rust/myfsio-engine/crates/myfsio-server/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "myfsio-server"
version = "0.1.0"
edition = "2021"
version.workspace = true
edition.workspace = true
[dependencies]
myfsio-common = { path = "../myfsio-common" }
@@ -45,6 +45,9 @@ tera = { workspace = true }
cookie = { workspace = true }
subtle = { workspace = true }
clap = { workspace = true }
dotenvy = { workspace = true }
sysinfo = "0.32"
aes-gcm = { workspace = true }
[dev-dependencies]
tempfile = "3"

92
rust/myfsio-engine/crates/myfsio-server/src/config.rs
View File

@@ -4,6 +4,7 @@ use std::path::PathBuf;
#[derive(Debug, Clone)]
pub struct ServerConfig {
pub bind_addr: SocketAddr,
pub ui_bind_addr: SocketAddr,
pub storage_root: PathBuf,
pub region: String,
pub iam_config_path: PathBuf,
@@ -16,6 +17,11 @@ pub struct ServerConfig {
pub gc_enabled: bool,
pub integrity_enabled: bool,
pub metrics_enabled: bool,
pub metrics_history_enabled: bool,
pub metrics_interval_minutes: u64,
pub metrics_retention_hours: u64,
pub metrics_history_interval_minutes: u64,
pub metrics_history_retention_hours: u64,
pub lifecycle_enabled: bool,
pub website_hosting_enabled: bool,
pub replication_connect_timeout_secs: u64,
@@ -42,22 +48,28 @@ impl ServerConfig {
.unwrap_or_else(|_| "5000".to_string())
.parse()
.unwrap_or(5000);
let storage_root = std::env::var("STORAGE_ROOT")
.unwrap_or_else(|_| "./data".to_string());
let region = std::env::var("AWS_REGION")
.unwrap_or_else(|_| "us-east-1".to_string());
let ui_port: u16 = std::env::var("UI_PORT")
.unwrap_or_else(|_| "5100".to_string())
.parse()
.unwrap_or(5100);
let storage_root = std::env::var("STORAGE_ROOT").unwrap_or_else(|_| "./data".to_string());
let region = std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string());
let storage_path = PathBuf::from(&storage_root);
let iam_config_path = std::env::var("IAM_CONFIG")
.map(PathBuf::from)
.unwrap_or_else(|_| {
storage_path.join(".myfsio.sys").join("config").join("iam.json")
storage_path
.join(".myfsio.sys")
.join("config")
.join("iam.json")
});
let sigv4_timestamp_tolerance_secs: u64 = std::env::var("SIGV4_TIMESTAMP_TOLERANCE_SECONDS")
.unwrap_or_else(|_| "900".to_string())
.parse()
.unwrap_or(900);
let sigv4_timestamp_tolerance_secs: u64 =
std::env::var("SIGV4_TIMESTAMP_TOLERANCE_SECONDS")
.unwrap_or_else(|_| "900".to_string())
.parse()
.unwrap_or(900);
let presigned_url_min_expiry: u64 = std::env::var("PRESIGNED_URL_MIN_EXPIRY_SECONDS")
.unwrap_or_else(|_| "1".to_string())
@@ -78,40 +90,60 @@ impl ServerConfig {
.join(".myfsio.sys")
.join("config")
.join(".secret");
std::fs::read_to_string(&secret_file).ok().map(|s| s.trim().to_string())
std::fs::read_to_string(&secret_file)
.ok()
.map(|s| s.trim().to_string())
}
}
};
let encryption_enabled = std::env::var("ENCRYPTION_ENABLED")
.unwrap_or_else(|_| "false".to_string())
.to_lowercase() == "true";
.to_lowercase()
== "true";
let kms_enabled = std::env::var("KMS_ENABLED")
.unwrap_or_else(|_| "false".to_string())
.to_lowercase() == "true";
.to_lowercase()
== "true";
let gc_enabled = std::env::var("GC_ENABLED")
.unwrap_or_else(|_| "false".to_string())
.to_lowercase() == "true";
.to_lowercase()
== "true";
let integrity_enabled = std::env::var("INTEGRITY_ENABLED")
.unwrap_or_else(|_| "false".to_string())
.to_lowercase() == "true";
.to_lowercase()
== "true";
let metrics_enabled = std::env::var("OPERATION_METRICS_ENABLED")
.unwrap_or_else(|_| "false".to_string())
.to_lowercase() == "true";
.to_lowercase()
== "true";
let metrics_history_enabled = std::env::var("METRICS_HISTORY_ENABLED")
.unwrap_or_else(|_| "false".to_string())
.to_lowercase()
== "true";
let metrics_interval_minutes = parse_u64_env("OPERATION_METRICS_INTERVAL_MINUTES", 5);
let metrics_retention_hours = parse_u64_env("OPERATION_METRICS_RETENTION_HOURS", 24);
let metrics_history_interval_minutes = parse_u64_env("METRICS_HISTORY_INTERVAL_MINUTES", 5);
let metrics_history_retention_hours = parse_u64_env("METRICS_HISTORY_RETENTION_HOURS", 24);
let lifecycle_enabled = std::env::var("LIFECYCLE_ENABLED")
.unwrap_or_else(|_| "false".to_string())
.to_lowercase() == "true";
.to_lowercase()
== "true";
let website_hosting_enabled = std::env::var("WEBSITE_HOSTING_ENABLED")
.unwrap_or_else(|_| "false".to_string())
.to_lowercase() == "true";
.to_lowercase()
== "true";
let replication_connect_timeout_secs = parse_u64_env("REPLICATION_CONNECT_TIMEOUT_SECONDS", 5);
let replication_connect_timeout_secs =
parse_u64_env("REPLICATION_CONNECT_TIMEOUT_SECONDS", 5);
let replication_read_timeout_secs = parse_u64_env("REPLICATION_READ_TIMEOUT_SECONDS", 30);
let replication_max_retries = parse_u64_env("REPLICATION_MAX_RETRIES", 2) as u32;
let replication_streaming_threshold_bytes =
@@ -121,20 +153,23 @@ impl ServerConfig {
let site_sync_enabled = std::env::var("SITE_SYNC_ENABLED")
.unwrap_or_else(|_| "false".to_string())
.to_lowercase() == "true";
.to_lowercase()
== "true";
let site_sync_interval_secs = parse_u64_env("SITE_SYNC_INTERVAL_SECONDS", 60);
let site_sync_batch_size = parse_u64_env("SITE_SYNC_BATCH_SIZE", 100) as usize;
let site_sync_connect_timeout_secs = parse_u64_env("SITE_SYNC_CONNECT_TIMEOUT_SECONDS", 10);
let site_sync_read_timeout_secs = parse_u64_env("SITE_SYNC_READ_TIMEOUT_SECONDS", 120);
let site_sync_max_retries = parse_u64_env("SITE_SYNC_MAX_RETRIES", 2) as u32;
let site_sync_clock_skew_tolerance: f64 = std::env::var("SITE_SYNC_CLOCK_SKEW_TOLERANCE_SECONDS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(1.0);
let site_sync_clock_skew_tolerance: f64 =
std::env::var("SITE_SYNC_CLOCK_SKEW_TOLERANCE_SECONDS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(1.0);
let ui_enabled = std::env::var("UI_ENABLED")
.unwrap_or_else(|_| "true".to_string())
.to_lowercase() == "true";
.to_lowercase()
== "true";
let templates_dir = std::env::var("TEMPLATES_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| default_templates_dir());
@@ -142,8 +177,10 @@ impl ServerConfig {
.map(PathBuf::from)
.unwrap_or_else(|_| default_static_dir());
let host_ip: std::net::IpAddr = host.parse().unwrap();
Self {
bind_addr: SocketAddr::new(host.parse().unwrap(), port),
bind_addr: SocketAddr::new(host_ip, port),
ui_bind_addr: SocketAddr::new(host_ip, ui_port),
storage_root: storage_path,
region,
iam_config_path,
@@ -156,6 +193,11 @@ impl ServerConfig {
gc_enabled,
integrity_enabled,
metrics_enabled,
metrics_history_enabled,
metrics_interval_minutes,
metrics_retention_hours,
metrics_history_interval_minutes,
metrics_history_retention_hours,
lifecycle_enabled,
website_hosting_enabled,
replication_connect_timeout_secs,

951
rust/myfsio-engine/crates/myfsio-server/src/handlers/admin.rs
View File

File diff suppressed because it is too large Load Diff

6
rust/myfsio-engine/crates/myfsio-server/src/handlers/chunked.rs
View File

@@ -41,7 +41,10 @@ impl<S> AwsChunkedStream<S> {
fn parse_chunk_size(line: &[u8]) -> std::io::Result<u64> {
let text = std::str::from_utf8(line).map_err(|_| {
std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid chunk size encoding")
std::io::Error::new(
std::io::ErrorKind::InvalidData,
"invalid chunk size encoding",
)
})?;
let head = text.split(';').next().unwrap_or("").trim();
u64::from_str_radix(head, 16).map_err(|_| {
@@ -179,4 +182,3 @@ pub fn decode_body(body: axum::body::Body) -> impl AsyncRead + Send + Unpin {
);
AwsChunkedStream::new(stream)
}

379
rust/myfsio-engine/crates/myfsio-server/src/handlers/config.rs
View File

@@ -13,8 +13,14 @@ fn xml_response(status: StatusCode, xml: String) -> Response {
fn storage_err(err: myfsio_storage::error::StorageError) -> Response {
let s3err = S3Error::from(err);
let status = StatusCode::from_u16(s3err.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
(status, [("content-type", "application/xml")], s3err.to_xml()).into_response()
let status =
StatusCode::from_u16(s3err.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
(
status,
[("content-type", "application/xml")],
s3err.to_xml(),
)
.into_response()
}
fn json_response(status: StatusCode, value: serde_json::Value) -> Response {
@@ -68,7 +74,7 @@ pub async fn get_tagging(state: &AppState, bucket: &str) -> Response {
Ok(config) => {
let mut xml = String::from(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<Tagging xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><TagSet>"
<Tagging xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><TagSet>",
);
for tag in &config.tags {
xml.push_str(&format!(
@@ -130,7 +136,11 @@ pub async fn get_cors(state: &AppState, bucket: &str) -> Response {
} else {
xml_response(
StatusCode::NOT_FOUND,
S3Error::new(S3ErrorCode::NoSuchKey, "The CORS configuration does not exist").to_xml(),
S3Error::new(
S3ErrorCode::NoSuchKey,
"The CORS configuration does not exist",
)
.to_xml(),
)
}
}
@@ -192,7 +202,8 @@ pub async fn get_encryption(state: &AppState, bucket: &str) -> Response {
S3Error::new(
S3ErrorCode::InvalidRequest,
"The server side encryption configuration was not found",
).to_xml(),
)
.to_xml(),
)
}
}
@@ -240,7 +251,11 @@ pub async fn get_lifecycle(state: &AppState, bucket: &str) -> Response {
} else {
xml_response(
StatusCode::NOT_FOUND,
S3Error::new(S3ErrorCode::NoSuchKey, "The lifecycle configuration does not exist").to_xml(),
S3Error::new(
S3ErrorCode::NoSuchKey,
"The lifecycle configuration does not exist",
)
.to_xml(),
)
}
}
@@ -328,17 +343,17 @@ pub async fn put_quota(state: &AppState, bucket: &str, body: Body) -> Response {
Err(_) => {
return xml_response(
StatusCode::BAD_REQUEST,
S3Error::new(S3ErrorCode::InvalidArgument, "Request body must be valid JSON").to_xml(),
S3Error::new(
S3ErrorCode::InvalidArgument,
"Request body must be valid JSON",
)
.to_xml(),
);
}
};
let max_size = payload
.get("max_size_bytes")
.and_then(|v| v.as_u64());
let max_objects = payload
.get("max_objects")
.and_then(|v| v.as_u64());
let max_size = payload.get("max_size_bytes").and_then(|v| v.as_u64());
let max_objects = payload.get("max_objects").and_then(|v| v.as_u64());
if max_size.is_none() && max_objects.is_none() {
return xml_response(
@@ -603,7 +618,11 @@ pub async fn get_website(state: &AppState, bucket: &str) -> Response {
} else {
xml_response(
StatusCode::NOT_FOUND,
S3Error::new(S3ErrorCode::NoSuchKey, "The website configuration does not exist").to_xml(),
S3Error::new(
S3ErrorCode::NoSuchKey,
"The website configuration does not exist",
)
.to_xml(),
)
}
}
@@ -677,19 +696,120 @@ pub async fn get_notification(state: &AppState, bucket: &str) -> Response {
}
pub async fn get_logging(state: &AppState, bucket: &str) -> Response {
match state.storage.get_bucket_config(bucket).await {
Ok(config) => {
if let Some(l) = &config.logging {
xml_response(StatusCode::OK, l.to_string())
} else {
let xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<BucketLoggingStatus xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
</BucketLoggingStatus>";
xml_response(StatusCode::OK, xml.to_string())
}
match state.storage.bucket_exists(bucket).await {
Ok(true) => {}
Ok(false) => {
return storage_err(myfsio_storage::error::StorageError::BucketNotFound(
bucket.to_string(),
))
}
Err(e) => storage_err(e),
Err(e) => return storage_err(e),
}
let logging_config = if let Some(cfg) = state.access_logging.get(bucket) {
Some(cfg)
} else {
match state.storage.get_bucket_config(bucket).await {
Ok(config) => {
let legacy = legacy_logging_config(&config);
if let Some(cfg) = legacy.as_ref() {
if let Err(err) = state.access_logging.set(bucket, cfg.clone()) {
tracing::warn!(
"Failed to migrate legacy bucket logging config for {}: {}",
bucket,
err
);
}
}
legacy
}
Err(e) => return storage_err(e),
}
};
let body = match logging_config {
Some(cfg) if cfg.enabled => format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<BucketLoggingStatus xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
<LoggingEnabled><TargetBucket>{}</TargetBucket><TargetPrefix>{}</TargetPrefix></LoggingEnabled>\
</BucketLoggingStatus>",
xml_escape(&cfg.target_bucket),
xml_escape(&cfg.target_prefix),
),
_ => "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<BucketLoggingStatus xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"></BucketLoggingStatus>"
.to_string(),
};
xml_response(StatusCode::OK, body)
}
fn xml_escape(s: &str) -> String {
s.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('>', "&gt;")
.replace('"', "&quot;")
.replace('\'', "&apos;")
}
fn legacy_logging_config(
config: &myfsio_common::types::BucketConfig,
) -> Option<crate::services::access_logging::LoggingConfiguration> {
let value = config.logging.as_ref()?;
match value {
serde_json::Value::String(xml) => parse_logging_config_xml(xml),
serde_json::Value::Object(_) => parse_logging_config_value(value.clone()),
_ => None,
}
}
fn parse_logging_config_value(
value: serde_json::Value,
) -> Option<crate::services::access_logging::LoggingConfiguration> {
let logging_enabled = value.get("LoggingEnabled")?;
let target_bucket = logging_enabled
.get("TargetBucket")
.and_then(|value| value.as_str())
.map(str::trim)
.filter(|value| !value.is_empty())?
.to_string();
let target_prefix = logging_enabled
.get("TargetPrefix")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string();
Some(crate::services::access_logging::LoggingConfiguration {
target_bucket,
target_prefix,
enabled: true,
})
}
fn parse_logging_config_xml(
xml: &str,
) -> Option<crate::services::access_logging::LoggingConfiguration> {
let doc = roxmltree::Document::parse(xml).ok()?;
let root = doc.root_element();
let logging_enabled = root
.children()
.find(|n| n.is_element() && n.tag_name().name() == "LoggingEnabled")?;
let target_bucket = logging_enabled
.children()
.find(|n| n.is_element() && n.tag_name().name() == "TargetBucket")
.and_then(|n| n.text())
.map(str::trim)
.filter(|value| !value.is_empty())?
.to_string();
let target_prefix = logging_enabled
.children()
.find(|n| n.is_element() && n.tag_name().name() == "TargetPrefix")
.and_then(|n| n.text())
.unwrap_or_default()
.to_string();
Some(crate::services::access_logging::LoggingConfiguration {
target_bucket,
target_prefix,
enabled: true,
})
}
pub async fn put_object_lock(state: &AppState, bucket: &str, body: Body) -> Response {
@@ -757,35 +877,125 @@ pub async fn delete_notification(state: &AppState, bucket: &str) -> Response {
}
pub async fn put_logging(state: &AppState, bucket: &str, body: Body) -> Response {
match state.storage.bucket_exists(bucket).await {
Ok(true) => {}
Ok(false) => {
return storage_err(myfsio_storage::error::StorageError::BucketNotFound(
bucket.to_string(),
))
}
Err(e) => return storage_err(e),
}
let body_bytes = match http_body_util::BodyExt::collect(body).await {
Ok(collected) => collected.to_bytes(),
Err(_) => return StatusCode::BAD_REQUEST.into_response(),
};
let value = serde_json::Value::String(String::from_utf8_lossy(&body_bytes).to_string());
match state.storage.get_bucket_config(bucket).await {
Ok(mut config) => {
config.logging = Some(value);
match state.storage.set_bucket_config(bucket, &config).await {
Ok(()) => StatusCode::OK.into_response(),
Err(e) => storage_err(e),
}
}
Err(e) => storage_err(e),
if body_bytes.iter().all(u8::is_ascii_whitespace) {
state.access_logging.delete(bucket);
return StatusCode::OK.into_response();
}
let xml = match std::str::from_utf8(&body_bytes) {
Ok(s) => s,
Err(_) => {
return s3_error_response(
S3ErrorCode::MalformedXML,
"Unable to parse XML document",
StatusCode::BAD_REQUEST,
)
}
};
let doc = match roxmltree::Document::parse(xml) {
Ok(d) => d,
Err(_) => {
return s3_error_response(
S3ErrorCode::MalformedXML,
"Unable to parse XML document",
StatusCode::BAD_REQUEST,
)
}
};
let root = doc.root_element();
let logging_enabled = root
.children()
.find(|n| n.is_element() && n.tag_name().name() == "LoggingEnabled");
let Some(le) = logging_enabled else {
state.access_logging.delete(bucket);
return StatusCode::OK.into_response();
};
let target_bucket = le
.children()
.find(|n| n.is_element() && n.tag_name().name() == "TargetBucket")
.and_then(|n| n.text())
.map(str::trim)
.unwrap_or_default();
if target_bucket.is_empty() {
return s3_error_response(
S3ErrorCode::InvalidArgument,
"TargetBucket is required",
StatusCode::BAD_REQUEST,
);
}
let cfg = crate::services::access_logging::LoggingConfiguration {
target_bucket: target_bucket.to_string(),
target_prefix: le
.children()
.find(|n| n.is_element() && n.tag_name().name() == "TargetPrefix")
.and_then(|n| n.text())
.unwrap_or_default()
.to_string(),
enabled: true,
};
match state.storage.bucket_exists(&cfg.target_bucket).await {
Ok(true) => {}
Ok(false) => {
return s3_error_response(
S3ErrorCode::InvalidArgument,
"Target bucket does not exist",
StatusCode::BAD_REQUEST,
)
}
Err(e) => return storage_err(e),
}
if let Err(e) = state.access_logging.set(bucket, cfg) {
tracing::error!(
"Failed to persist bucket logging config for {}: {}",
bucket,
e
);
return StatusCode::INTERNAL_SERVER_ERROR.into_response();
}
StatusCode::OK.into_response()
}
pub async fn delete_logging(state: &AppState, bucket: &str) -> Response {
match state.storage.get_bucket_config(bucket).await {
Ok(mut config) => {
config.logging = None;
match state.storage.set_bucket_config(bucket, &config).await {
Ok(()) => StatusCode::NO_CONTENT.into_response(),
Err(e) => storage_err(e),
}
match state.storage.bucket_exists(bucket).await {
Ok(true) => {}
Ok(false) => {
return storage_err(myfsio_storage::error::StorageError::BucketNotFound(
bucket.to_string(),
))
}
Err(e) => storage_err(e),
Err(e) => return storage_err(e),
}
state.access_logging.delete(bucket);
StatusCode::NO_CONTENT.into_response()
}
fn s3_error_response(code: S3ErrorCode, message: &str, status: StatusCode) -> Response {
let err = S3Error::new(code, message.to_string());
(status, [("content-type", "application/xml")], err.to_xml()).into_response()
}
pub async fn list_object_versions(state: &AppState, bucket: &str) -> Response {
@@ -812,7 +1022,7 @@ pub async fn list_object_versions(state: &AppState, bucket: &str) -> Response {
let mut xml = String::from(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<ListVersionsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
<ListVersionsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">",
);
xml.push_str(&format!("<Name>{}</Name>", bucket));
@@ -842,7 +1052,7 @@ pub async fn get_object_tagging(state: &AppState, bucket: &str, key: &str) -> Re
Ok(tags) => {
let mut xml = String::from(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<Tagging xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><TagSet>"
<Tagging xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><TagSet>",
);
for tag in &tags {
xml.push_str(&format!(
@@ -910,20 +1120,24 @@ pub async fn put_object_acl(state: &AppState, bucket: &str, key: &str, _body: Bo
pub async fn get_object_retention(state: &AppState, bucket: &str, key: &str) -> Response {
match state.storage.head_object(bucket, key).await {
Ok(_) => {
xml_response(
StatusCode::NOT_FOUND,
S3Error::new(
S3ErrorCode::InvalidRequest,
"No retention policy configured",
).to_xml(),
Ok(_) => xml_response(
StatusCode::NOT_FOUND,
S3Error::new(
S3ErrorCode::InvalidRequest,
"No retention policy configured",
)
}
.to_xml(),
),
Err(e) => storage_err(e),
}
}
pub async fn put_object_retention(state: &AppState, bucket: &str, key: &str, _body: Body) -> Response {
pub async fn put_object_retention(
state: &AppState,
bucket: &str,
key: &str,
_body: Body,
) -> Response {
match state.storage.head_object(bucket, key).await {
Ok(_) => StatusCode::OK.into_response(),
Err(e) => storage_err(e),
@@ -942,13 +1156,68 @@ pub async fn get_object_legal_hold(state: &AppState, bucket: &str, key: &str) ->
}
}
pub async fn put_object_legal_hold(state: &AppState, bucket: &str, key: &str, _body: Body) -> Response {
pub async fn put_object_legal_hold(
state: &AppState,
bucket: &str,
key: &str,
_body: Body,
) -> Response {
match state.storage.head_object(bucket, key).await {
Ok(_) => StatusCode::OK.into_response(),
Err(e) => storage_err(e),
}
}
#[cfg(test)]
mod tests {
use super::{legacy_logging_config, parse_logging_config_xml};
use myfsio_common::types::BucketConfig;
#[test]
fn parses_legacy_logging_xml_string() {
let mut config = BucketConfig::default();
config.logging = Some(serde_json::Value::String(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<BucketLoggingStatus xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
<LoggingEnabled><TargetBucket>logs</TargetBucket><TargetPrefix>audit/</TargetPrefix></LoggingEnabled>\
</BucketLoggingStatus>"
.to_string(),
));
let parsed = legacy_logging_config(&config).expect("expected legacy logging config");
assert_eq!(parsed.target_bucket, "logs");
assert_eq!(parsed.target_prefix, "audit/");
assert!(parsed.enabled);
}
#[test]
fn parses_legacy_logging_json_object() {
let mut config = BucketConfig::default();
config.logging = Some(serde_json::json!({
"LoggingEnabled": {
"TargetBucket": "logs",
"TargetPrefix": "archive/"
}
}));
let parsed = legacy_logging_config(&config).expect("expected legacy logging config");
assert_eq!(parsed.target_bucket, "logs");
assert_eq!(parsed.target_prefix, "archive/");
assert!(parsed.enabled);
}
#[test]
fn ignores_logging_xml_without_enabled_block() {
let parsed = parse_logging_config_xml(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<BucketLoggingStatus xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
</BucketLoggingStatus>",
);
assert!(parsed.is_none());
}
}
fn parse_tagging_xml(xml: &str) -> Vec<myfsio_common::types::Tag> {
let mut tags = Vec::new();
let mut in_tag = false;

473
rust/myfsio-engine/crates/myfsio-server/src/handlers/kms.rs
View File

@@ -1,14 +1,17 @@
use aes_gcm::aead::Aead;
use aes_gcm::{Aes256Gcm, KeyInit, Nonce};
use axum::body::Body;
use axum::extract::State;
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use base64::engine::general_purpose::STANDARD as B64;
use base64::Engine;
use serde_json::json;
use rand::RngCore;
use serde_json::{json, Value};
use crate::state::AppState;
fn json_ok(value: serde_json::Value) -> Response {
fn json_ok(value: Value) -> Response {
(
StatusCode::OK,
[("content-type", "application/json")],
@@ -26,14 +29,54 @@ fn json_err(status: StatusCode, msg: &str) -> Response {
.into_response()
}
async fn read_json(body: Body) -> Result<Value, Response> {
let body_bytes = http_body_util::BodyExt::collect(body)
.await
.map_err(|_| json_err(StatusCode::BAD_REQUEST, "Invalid request body"))?
.to_bytes();
if body_bytes.is_empty() {
Ok(json!({}))
} else {
serde_json::from_slice(&body_bytes)
.map_err(|_| json_err(StatusCode::BAD_REQUEST, "Invalid JSON"))
}
}
fn require_kms(
state: &AppState,
) -> Result<&std::sync::Arc<myfsio_crypto::kms::KmsService>, Response> {
state
.kms
.as_ref()
.ok_or_else(|| json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"))
}
fn decode_b64(value: &str, field: &str) -> Result<Vec<u8>, Response> {
B64.decode(value).map_err(|_| {
json_err(
StatusCode::BAD_REQUEST,
&format!("Invalid base64 {}", field),
)
})
}
fn require_str<'a>(value: &'a Value, names: &[&str], message: &str) -> Result<&'a str, Response> {
for name in names {
if let Some(found) = value.get(*name).and_then(|v| v.as_str()) {
return Ok(found);
}
}
Err(json_err(StatusCode::BAD_REQUEST, message))
}
pub async fn list_keys(State(state): State<AppState>) -> Response {
let kms = match &state.kms {
Some(k) => k,
None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"),
let kms = match require_kms(&state) {
Ok(kms) => kms,
Err(response) => return response,
};
let keys = kms.list_keys().await;
let keys_json: Vec<serde_json::Value> = keys
let keys_json: Vec<Value> = keys
.iter()
.map(|k| {
json!({
@@ -53,31 +96,22 @@ pub async fn list_keys(State(state): State<AppState>) -> Response {
}
pub async fn create_key(State(state): State<AppState>, body: Body) -> Response {
let kms = match &state.kms {
Some(k) => k,
None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"),
let kms = match require_kms(&state) {
Ok(kms) => kms,
Err(response) => return response,
};
let req = match read_json(body).await {
Ok(req) => req,
Err(response) => return response,
};
let body_bytes = match http_body_util::BodyExt::collect(body).await {
Ok(c) => c.to_bytes(),
Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid request body"),
};
let description = req
.get("Description")
.or_else(|| req.get("description"))
.and_then(|d| d.as_str())
.unwrap_or("");
let description = if body_bytes.is_empty() {
String::new()
} else {
match serde_json::from_slice::<serde_json::Value>(&body_bytes) {
Ok(v) => v
.get("Description")
.or_else(|| v.get("description"))
.and_then(|d| d.as_str())
.unwrap_or("")
.to_string(),
Err(_) => String::new(),
}
};
match kms.create_key(&description).await {
match kms.create_key(description).await {
Ok(key) => json_ok(json!({
"KeyId": key.key_id,
"Arn": key.arn,
@@ -94,9 +128,9 @@ pub async fn get_key(
State(state): State<AppState>,
axum::extract::Path(key_id): axum::extract::Path<String>,
) -> Response {
let kms = match &state.kms {
Some(k) => k,
None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"),
let kms = match require_kms(&state) {
Ok(kms) => kms,
Err(response) => return response,
};
match kms.get_key(&key_id).await {
@@ -118,9 +152,9 @@ pub async fn delete_key(
State(state): State<AppState>,
axum::extract::Path(key_id): axum::extract::Path<String>,
) -> Response {
let kms = match &state.kms {
Some(k) => k,
None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"),
let kms = match require_kms(&state) {
Ok(kms) => kms,
Err(response) => return response,
};
match kms.delete_key(&key_id).await {
@@ -134,9 +168,9 @@ pub async fn enable_key(
State(state): State<AppState>,
axum::extract::Path(key_id): axum::extract::Path<String>,
) -> Response {
let kms = match &state.kms {
Some(k) => k,
None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"),
let kms = match require_kms(&state) {
Ok(kms) => kms,
Err(response) => return response,
};
match kms.enable_key(&key_id).await {
@@ -150,9 +184,9 @@ pub async fn disable_key(
State(state): State<AppState>,
axum::extract::Path(key_id): axum::extract::Path<String>,
) -> Response {
let kms = match &state.kms {
Some(k) => k,
None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"),
let kms = match require_kms(&state) {
Ok(kms) => kms,
Err(response) => return response,
};
match kms.disable_key(&key_id).await {
@@ -163,32 +197,26 @@ pub async fn disable_key(
}
pub async fn encrypt(State(state): State<AppState>, body: Body) -> Response {
let kms = match &state.kms {
Some(k) => k,
None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"),
let kms = match require_kms(&state) {
Ok(kms) => kms,
Err(response) => return response,
};
let req = match read_json(body).await {
Ok(req) => req,
Err(response) => return response,
};
let body_bytes = match http_body_util::BodyExt::collect(body).await {
Ok(c) => c.to_bytes(),
Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid request body"),
let key_id = match require_str(&req, &["KeyId", "key_id"], "Missing KeyId") {
Ok(value) => value,
Err(response) => return response,
};
let req: serde_json::Value = match serde_json::from_slice(&body_bytes) {
Ok(v) => v,
Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid JSON"),
let plaintext_b64 = match require_str(&req, &["Plaintext", "plaintext"], "Missing Plaintext") {
Ok(value) => value,
Err(response) => return response,
};
let key_id = match req.get("KeyId").and_then(|v| v.as_str()) {
Some(k) => k,
None => return json_err(StatusCode::BAD_REQUEST, "Missing KeyId"),
};
let plaintext_b64 = match req.get("Plaintext").and_then(|v| v.as_str()) {
Some(p) => p,
None => return json_err(StatusCode::BAD_REQUEST, "Missing Plaintext"),
};
let plaintext = match B64.decode(plaintext_b64) {
Ok(p) => p,
Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid base64 Plaintext"),
let plaintext = match decode_b64(plaintext_b64, "Plaintext") {
Ok(value) => value,
Err(response) => return response,
};
match kms.encrypt_data(key_id, &plaintext).await {
@@ -201,32 +229,30 @@ pub async fn encrypt(State(state): State<AppState>, body: Body) -> Response {
}
pub async fn decrypt(State(state): State<AppState>, body: Body) -> Response {
let kms = match &state.kms {
Some(k) => k,
None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"),
let kms = match require_kms(&state) {
Ok(kms) => kms,
Err(response) => return response,
};
let req = match read_json(body).await {
Ok(req) => req,
Err(response) => return response,
};
let body_bytes = match http_body_util::BodyExt::collect(body).await {
Ok(c) => c.to_bytes(),
Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid request body"),
let key_id = match require_str(&req, &["KeyId", "key_id"], "Missing KeyId") {
Ok(value) => value,
Err(response) => return response,
};
let req: serde_json::Value = match serde_json::from_slice(&body_bytes) {
Ok(v) => v,
Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid JSON"),
let ciphertext_b64 = match require_str(
&req,
&["CiphertextBlob", "ciphertext_blob"],
"Missing CiphertextBlob",
) {
Ok(value) => value,
Err(response) => return response,
};
let key_id = match req.get("KeyId").and_then(|v| v.as_str()) {
Some(k) => k,
None => return json_err(StatusCode::BAD_REQUEST, "Missing KeyId"),
};
let ct_b64 = match req.get("CiphertextBlob").and_then(|v| v.as_str()) {
Some(c) => c,
None => return json_err(StatusCode::BAD_REQUEST, "Missing CiphertextBlob"),
};
let ciphertext = match B64.decode(ct_b64) {
Ok(c) => c,
Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid base64"),
let ciphertext = match decode_b64(ciphertext_b64, "CiphertextBlob") {
Ok(value) => value,
Err(response) => return response,
};
match kms.decrypt_data(key_id, &ciphertext).await {
@@ -239,39 +265,276 @@ pub async fn decrypt(State(state): State<AppState>, body: Body) -> Response {
}
pub async fn generate_data_key(State(state): State<AppState>, body: Body) -> Response {
let kms = match &state.kms {
Some(k) => k,
None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"),
generate_data_key_inner(state, body, true).await
}
pub async fn generate_data_key_without_plaintext(
State(state): State<AppState>,
body: Body,
) -> Response {
generate_data_key_inner(state, body, false).await
}
async fn generate_data_key_inner(state: AppState, body: Body, include_plaintext: bool) -> Response {
let kms = match require_kms(&state) {
Ok(kms) => kms,
Err(response) => return response,
};
let req = match read_json(body).await {
Ok(req) => req,
Err(response) => return response,
};
let body_bytes = match http_body_util::BodyExt::collect(body).await {
Ok(c) => c.to_bytes(),
Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid request body"),
};
let req: serde_json::Value = match serde_json::from_slice(&body_bytes) {
Ok(v) => v,
Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid JSON"),
};
let key_id = match req.get("KeyId").and_then(|v| v.as_str()) {
Some(k) => k,
None => return json_err(StatusCode::BAD_REQUEST, "Missing KeyId"),
let key_id = match require_str(&req, &["KeyId", "key_id"], "Missing KeyId") {
Ok(value) => value,
Err(response) => return response,
};
let num_bytes = req
.get("NumberOfBytes")
.and_then(|v| v.as_u64())
.unwrap_or(32) as usize;
if num_bytes < 1 || num_bytes > 1024 {
if !(1..=1024).contains(&num_bytes) {
return json_err(StatusCode::BAD_REQUEST, "NumberOfBytes must be 1-1024");
}
match kms.generate_data_key(key_id, num_bytes).await {
Ok((plaintext, wrapped)) => json_ok(json!({
"KeyId": key_id,
"Plaintext": B64.encode(&plaintext),
"CiphertextBlob": B64.encode(&wrapped),
Ok((plaintext, wrapped)) => {
let mut value = json!({
"KeyId": key_id,
"CiphertextBlob": B64.encode(&wrapped),
});
if include_plaintext {
value["Plaintext"] = json!(B64.encode(&plaintext));
}
json_ok(value)
}
Err(e) => json_err(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()),
}
}
pub async fn re_encrypt(State(state): State<AppState>, body: Body) -> Response {
let kms = match require_kms(&state) {
Ok(kms) => kms,
Err(response) => return response,
};
let req = match read_json(body).await {
Ok(req) => req,
Err(response) => return response,
};
let ciphertext_b64 = match require_str(
&req,
&["CiphertextBlob", "ciphertext_blob"],
"CiphertextBlob is required",
) {
Ok(value) => value,
Err(response) => return response,
};
let destination_key_id = match require_str(
&req,
&["DestinationKeyId", "destination_key_id"],
"DestinationKeyId is required",
) {
Ok(value) => value,
Err(response) => return response,
};
let ciphertext = match decode_b64(ciphertext_b64, "CiphertextBlob") {
Ok(value) => value,
Err(response) => return response,
};
let keys = kms.list_keys().await;
let mut source_key_id: Option<String> = None;
let mut plaintext: Option<Vec<u8>> = None;
for key in keys {
if !key.enabled {
continue;
}
if let Ok(value) = kms.decrypt_data(&key.key_id, &ciphertext).await {
source_key_id = Some(key.key_id);
plaintext = Some(value);
break;
}
}
let Some(source_key_id) = source_key_id else {
return json_err(
StatusCode::BAD_REQUEST,
"Could not determine source key for CiphertextBlob",
);
};
let plaintext = plaintext.unwrap_or_default();
match kms.encrypt_data(destination_key_id, &plaintext).await {
Ok(new_ciphertext) => json_ok(json!({
"CiphertextBlob": B64.encode(&new_ciphertext),
"SourceKeyId": source_key_id,
"KeyId": destination_key_id,
})),
Err(e) => json_err(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()),
}
}
pub async fn generate_random(State(state): State<AppState>, body: Body) -> Response {
if let Err(response) = require_kms(&state) {
return response;
}
let req = match read_json(body).await {
Ok(req) => req,
Err(response) => return response,
};
let num_bytes = req
.get("NumberOfBytes")
.and_then(|v| v.as_u64())
.unwrap_or(32) as usize;
if !(1..=1024).contains(&num_bytes) {
return json_err(StatusCode::BAD_REQUEST, "NumberOfBytes must be 1-1024");
}
let mut bytes = vec![0u8; num_bytes];
rand::thread_rng().fill_bytes(&mut bytes);
json_ok(json!({
"Plaintext": B64.encode(bytes),
}))
}
pub async fn client_generate_key(State(state): State<AppState>) -> Response {
let _ = state;
let mut key = [0u8; 32];
rand::thread_rng().fill_bytes(&mut key);
json_ok(json!({
"Key": B64.encode(key),
"Algorithm": "AES-256-GCM",
"KeySize": 32,
}))
}
pub async fn client_encrypt(State(state): State<AppState>, body: Body) -> Response {
let _ = state;
let req = match read_json(body).await {
Ok(req) => req,
Err(response) => return response,
};
let plaintext_b64 =
match require_str(&req, &["Plaintext", "plaintext"], "Plaintext is required") {
Ok(value) => value,
Err(response) => return response,
};
let key_b64 = match require_str(&req, &["Key", "key"], "Key is required") {
Ok(value) => value,
Err(response) => return response,
};
let plaintext = match decode_b64(plaintext_b64, "Plaintext") {
Ok(value) => value,
Err(response) => return response,
};
let key_bytes = match decode_b64(key_b64, "Key") {
Ok(value) => value,
Err(response) => return response,
};
if key_bytes.len() != 32 {
return json_err(StatusCode::BAD_REQUEST, "Key must decode to 32 bytes");
}
let cipher = match Aes256Gcm::new_from_slice(&key_bytes) {
Ok(cipher) => cipher,
Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid encryption key"),
};
let mut nonce_bytes = [0u8; 12];
rand::thread_rng().fill_bytes(&mut nonce_bytes);
let nonce = Nonce::from_slice(&nonce_bytes);
match cipher.encrypt(nonce, plaintext.as_ref()) {
Ok(ciphertext) => json_ok(json!({
"Ciphertext": B64.encode(ciphertext),
"Nonce": B64.encode(nonce_bytes),
"Algorithm": "AES-256-GCM",
})),
Err(e) => json_err(StatusCode::BAD_REQUEST, &e.to_string()),
}
}
pub async fn client_decrypt(State(state): State<AppState>, body: Body) -> Response {
let _ = state;
let req = match read_json(body).await {
Ok(req) => req,
Err(response) => return response,
};
let ciphertext_b64 = match require_str(
&req,
&["Ciphertext", "ciphertext"],
"Ciphertext is required",
) {
Ok(value) => value,
Err(response) => return response,
};
let nonce_b64 = match require_str(&req, &["Nonce", "nonce"], "Nonce is required") {
Ok(value) => value,
Err(response) => return response,
};
let key_b64 = match require_str(&req, &["Key", "key"], "Key is required") {
Ok(value) => value,
Err(response) => return response,
};
let ciphertext = match decode_b64(ciphertext_b64, "Ciphertext") {
Ok(value) => value,
Err(response) => return response,
};
let nonce_bytes = match decode_b64(nonce_b64, "Nonce") {
Ok(value) => value,
Err(response) => return response,
};
let key_bytes = match decode_b64(key_b64, "Key") {
Ok(value) => value,
Err(response) => return response,
};
if key_bytes.len() != 32 {
return json_err(StatusCode::BAD_REQUEST, "Key must decode to 32 bytes");
}
if nonce_bytes.len() != 12 {
return json_err(StatusCode::BAD_REQUEST, "Nonce must decode to 12 bytes");
}
let cipher = match Aes256Gcm::new_from_slice(&key_bytes) {
Ok(cipher) => cipher,
Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid encryption key"),
};
let nonce = Nonce::from_slice(&nonce_bytes);
match cipher.decrypt(nonce, ciphertext.as_ref()) {
Ok(plaintext) => json_ok(json!({
"Plaintext": B64.encode(plaintext),
})),
Err(e) => json_err(StatusCode::BAD_REQUEST, &e.to_string()),
}
}
pub async fn materials(
State(state): State<AppState>,
axum::extract::Path(key_id): axum::extract::Path<String>,
body: Body,
) -> Response {
let kms = match require_kms(&state) {
Ok(kms) => kms,
Err(response) => return response,
};
let _ = match read_json(body).await {
Ok(req) => req,
Err(response) => return response,
};
match kms.generate_data_key(&key_id, 32).await {
Ok((plaintext, wrapped)) => json_ok(json!({
"PlaintextKey": B64.encode(plaintext),
"EncryptedKey": B64.encode(wrapped),
"KeyId": key_id,
"Algorithm": "AES-256-GCM",
"KeyWrapAlgorithm": "kms",
})),
Err(e) => json_err(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()),
}

327
rust/myfsio-engine/crates/myfsio-server/src/handlers/mod.rs
View File

@@ -4,6 +4,7 @@ mod config;
pub mod kms;
mod select;
pub mod ui;
pub mod ui_api;
pub mod ui_pages;
use std::collections::HashMap;
@@ -15,6 +16,7 @@ use axum::response::{IntoResponse, Response};
use base64::engine::general_purpose::URL_SAFE;
use base64::Engine;
use chrono::{DateTime, Utc};
use serde_json::json;
use myfsio_common::error::{S3Error, S3ErrorCode};
use myfsio_common::types::PartInfo;
@@ -25,7 +27,8 @@ use tokio_util::io::ReaderStream;
use crate::state::AppState;
fn s3_error_response(err: S3Error) -> Response {
let status = StatusCode::from_u16(err.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
let status =
StatusCode::from_u16(err.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
let resource = if err.resource.is_empty() {
"/".to_string()
} else {
@@ -35,12 +38,7 @@ fn s3_error_response(err: S3Error) -> Response {
.with_resource(resource)
.with_request_id(uuid::Uuid::new_v4().simple().to_string())
.to_xml();
(
status,
[("content-type", "application/xml")],
body,
)
.into_response()
(status, [("content-type", "application/xml")], body).into_response()
}
fn storage_err_response(err: myfsio_storage::error::StorageError) -> Response {
@@ -51,17 +49,25 @@ pub async fn list_buckets(State(state): State<AppState>) -> Response {
match state.storage.list_buckets().await {
Ok(buckets) => {
let xml = myfsio_xml::response::list_buckets_xml("myfsio", "myfsio", &buckets);
(
StatusCode::OK,
[("content-type", "application/xml")],
xml,
)
.into_response()
(StatusCode::OK, [("content-type", "application/xml")], xml).into_response()
}
Err(e) => storage_err_response(e),
}
}
pub async fn health_check() -> Response {
(
StatusCode::OK,
[("content-type", "application/json")],
json!({
"status": "ok",
"version": env!("CARGO_PKG_VERSION"),
})
.to_string(),
)
.into_response()
}
pub async fn create_bucket(
State(state): State<AppState>,
Path(bucket): Path<String>,
@@ -109,14 +115,12 @@ pub async fn create_bucket(
}
match state.storage.create_bucket(&bucket).await {
Ok(()) => {
(
StatusCode::OK,
[("location", format!("/{}", bucket).as_str())],
"",
)
.into_response()
}
Ok(()) => (
StatusCode::OK,
[("location", format!("/{}", bucket).as_str())],
"",
)
.into_response(),
Err(e) => storage_err_response(e),
}
}
@@ -162,9 +166,7 @@ pub async fn get_bucket(
Query(query): Query<BucketQuery>,
) -> Response {
if !matches!(state.storage.bucket_exists(&bucket).await, Ok(true)) {
return storage_err_response(
myfsio_storage::error::StorageError::BucketNotFound(bucket),
);
return storage_err_response(myfsio_storage::error::StorageError::BucketNotFound(bucket));
}
if query.quota.is_some() {
@@ -258,8 +260,16 @@ pub async fn get_bucket(
let params = myfsio_common::types::ListParams {
max_keys,
continuation_token: effective_start.clone(),
prefix: if prefix.is_empty() { None } else { Some(prefix.clone()) },
start_after: if is_v2 { query.start_after.clone() } else { None },
prefix: if prefix.is_empty() {
None
} else {
Some(prefix.clone())
},
start_after: if is_v2 {
query.start_after.clone()
} else {
None
},
};
match state.storage.list_objects(&bucket, &params).await {
Ok(result) => {
@@ -411,19 +421,16 @@ pub async fn delete_bucket(
}
}
pub async fn head_bucket(
State(state): State<AppState>,
Path(bucket): Path<String>,
) -> Response {
pub async fn head_bucket(State(state): State<AppState>, Path(bucket): Path<String>) -> Response {
match state.storage.bucket_exists(&bucket).await {
Ok(true) => {
let mut headers = HeaderMap::new();
headers.insert("x-amz-bucket-region", state.config.region.parse().unwrap());
(StatusCode::OK, headers).into_response()
}
Ok(false) => storage_err_response(
myfsio_storage::error::StorageError::BucketNotFound(bucket),
),
Ok(false) => {
storage_err_response(myfsio_storage::error::StorageError::BucketNotFound(bucket))
}
Err(e) => storage_err_response(e),
}
}
@@ -458,22 +465,34 @@ pub struct ObjectQuery {
fn apply_response_overrides(headers: &mut HeaderMap, query: &ObjectQuery) {
if let Some(ref v) = query.response_content_type {
if let Ok(val) = v.parse() { headers.insert("content-type", val); }
if let Ok(val) = v.parse() {
headers.insert("content-type", val);
}
}
if let Some(ref v) = query.response_content_disposition {
if let Ok(val) = v.parse() { headers.insert("content-disposition", val); }
if let Ok(val) = v.parse() {
headers.insert("content-disposition", val);
}
}
if let Some(ref v) = query.response_content_language {
if let Ok(val) = v.parse() { headers.insert("content-language", val); }
if let Ok(val) = v.parse() {
headers.insert("content-language", val);
}
}
if let Some(ref v) = query.response_content_encoding {
if let Ok(val) = v.parse() { headers.insert("content-encoding", val); }
if let Ok(val) = v.parse() {
headers.insert("content-encoding", val);
}
}
if let Some(ref v) = query.response_cache_control {
if let Ok(val) = v.parse() { headers.insert("cache-control", val); }
if let Ok(val) = v.parse() {
headers.insert("cache-control", val);
}
}
if let Some(ref v) = query.response_expires {
if let Ok(val) = v.parse() { headers.insert("expires", val); }
if let Ok(val) = v.parse() {
headers.insert("expires", val);
}
}
}
@@ -490,12 +509,18 @@ fn guessed_content_type(key: &str, explicit: Option<&str>) -> String {
}
fn is_aws_chunked(headers: &HeaderMap) -> bool {
if let Some(enc) = headers.get("content-encoding").and_then(|v| v.to_str().ok()) {
if let Some(enc) = headers
.get("content-encoding")
.and_then(|v| v.to_str().ok())
{
if enc.to_ascii_lowercase().contains("aws-chunked") {
return true;
}
}
if let Some(sha) = headers.get("x-amz-content-sha256").and_then(|v| v.to_str().ok()) {
if let Some(sha) = headers
.get("x-amz-content-sha256")
.and_then(|v| v.to_str().ok())
{
let lower = sha.to_ascii_lowercase();
if lower.starts_with("streaming-") {
return true;
@@ -535,7 +560,10 @@ pub async fn put_object(
if let Some(ref upload_id) = query.upload_id {
if let Some(part_number) = query.part_number {
if let Some(copy_source) = headers.get("x-amz-copy-source").and_then(|v| v.to_str().ok()) {
if let Some(copy_source) = headers
.get("x-amz-copy-source")
.and_then(|v| v.to_str().ok())
{
let range = headers
.get("x-amz-copy-source-range")
.and_then(|v| v.to_str().ok());
@@ -562,15 +590,16 @@ pub async fn put_object(
}
}
if let Some(copy_source) = headers.get("x-amz-copy-source").and_then(|v| v.to_str().ok()) {
if let Some(copy_source) = headers
.get("x-amz-copy-source")
.and_then(|v| v.to_str().ok())
{
return copy_object_handler(&state, copy_source, &bucket, &key, &headers).await;
}
let content_type = guessed_content_type(
&key,
headers
.get("content-type")
.and_then(|v| v.to_str().ok()),
headers.get("content-type").and_then(|v| v.to_str().ok()),
);
let mut metadata = HashMap::new();
@@ -589,14 +618,18 @@ pub async fn put_object(
Box::pin(chunked::decode_body(body))
} else {
let stream = tokio_util::io::StreamReader::new(
http_body_util::BodyStream::new(body).map_ok(|frame| {
frame.into_data().unwrap_or_default()
}).map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)),
http_body_util::BodyStream::new(body)
.map_ok(|frame| frame.into_data().unwrap_or_default())
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)),
);
Box::pin(stream)
};
match state.storage.put_object(&bucket, &key, boxed, Some(metadata)).await {
match state
.storage
.put_object(&bucket, &key, boxed, Some(metadata))
.await
{
Ok(meta) => {
if let Some(enc_ctx) = resolve_encryption_context(&state, &bucket, &headers).await {
if let Some(ref enc_svc) = state.encryption {
@@ -612,26 +645,39 @@ pub async fn put_object(
Ok(enc_meta) => {
if let Err(e) = tokio::fs::rename(&enc_tmp, &obj_path).await {
let _ = tokio::fs::remove_file(&enc_tmp).await;
return storage_err_response(myfsio_storage::error::StorageError::Io(e));
return storage_err_response(
myfsio_storage::error::StorageError::Io(e),
);
}
let enc_size = tokio::fs::metadata(&obj_path).await.map(|m| m.len()).unwrap_or(0);
let enc_size = tokio::fs::metadata(&obj_path)
.await
.map(|m| m.len())
.unwrap_or(0);
let mut enc_metadata = enc_meta.to_metadata_map();
let all_meta = match state.storage.get_object_metadata(&bucket, &key).await {
Ok(m) => m,
Err(_) => HashMap::new(),
};
let all_meta =
match state.storage.get_object_metadata(&bucket, &key).await {
Ok(m) => m,
Err(_) => HashMap::new(),
};
for (k, v) in &all_meta {
enc_metadata.entry(k.clone()).or_insert_with(|| v.clone());
}
enc_metadata.insert("__size__".to_string(), enc_size.to_string());
let _ = state.storage.put_object_metadata(&bucket, &key, &enc_metadata).await;
let _ = state
.storage
.put_object_metadata(&bucket, &key, &enc_metadata)
.await;
let mut resp_headers = HeaderMap::new();
if let Some(ref etag) = meta.etag {
resp_headers.insert("etag", format!("\"{}\"", etag).parse().unwrap());
resp_headers
.insert("etag", format!("\"{}\"", etag).parse().unwrap());
}
resp_headers.insert("x-amz-server-side-encryption", enc_ctx.algorithm.as_str().parse().unwrap());
resp_headers.insert(
"x-amz-server-side-encryption",
enc_ctx.algorithm.as_str().parse().unwrap(),
);
return (StatusCode::OK, resp_headers).into_response();
}
Err(e) => {
@@ -697,7 +743,11 @@ pub async fn get_object(
return range_get_handler(&state, &bucket, &key, range_str, &query).await;
}
let all_meta = state.storage.get_object_metadata(&bucket, &key).await.unwrap_or_default();
let all_meta = state
.storage
.get_object_metadata(&bucket, &key)
.await
.unwrap_or_default();
let enc_meta = myfsio_crypto::encryption::EncryptionMetadata::from_metadata(&all_meta);
if let (Some(ref enc_info), Some(ref enc_svc)) = (&enc_meta, &state.encryption) {
@@ -712,7 +762,10 @@ pub async fn get_object(
let customer_key = extract_sse_c_key(&headers);
let ck_ref = customer_key.as_deref();
if let Err(e) = enc_svc.decrypt_object(&obj_path, &dec_tmp, enc_info, ck_ref).await {
if let Err(e) = enc_svc
.decrypt_object(&obj_path, &dec_tmp, enc_info, ck_ref)
.await
{
let _ = tokio::fs::remove_file(&dec_tmp).await;
return s3_error_response(S3Error::new(
myfsio_common::error::S3ErrorCode::InternalError,
@@ -747,10 +800,17 @@ pub async fn get_object(
insert_content_type(&mut resp_headers, &key, meta.content_type.as_deref());
resp_headers.insert(
"last-modified",
meta.last_modified.format("%a, %d %b %Y %H:%M:%S GMT").to_string().parse().unwrap(),
meta.last_modified
.format("%a, %d %b %Y %H:%M:%S GMT")
.to_string()
.parse()
.unwrap(),
);
resp_headers.insert("accept-ranges", "bytes".parse().unwrap());
resp_headers.insert("x-amz-server-side-encryption", enc_info.algorithm.parse().unwrap());
resp_headers.insert(
"x-amz-server-side-encryption",
enc_info.algorithm.parse().unwrap(),
);
for (k, v) in &meta.metadata {
if let Ok(header_val) = v.parse() {
@@ -889,11 +949,7 @@ pub async fn head_object(
}
}
async fn initiate_multipart_handler(
state: &AppState,
bucket: &str,
key: &str,
) -> Response {
async fn initiate_multipart_handler(state: &AppState, bucket: &str, key: &str) -> Response {
match state.storage.initiate_multipart(bucket, key, None).await {
Ok(upload_id) => {
let xml = myfsio_xml::response::initiate_multipart_upload_xml(bucket, key, &upload_id);
@@ -915,14 +971,18 @@ async fn upload_part_handler_with_chunking(
Box::pin(chunked::decode_body(body))
} else {
let stream = tokio_util::io::StreamReader::new(
http_body_util::BodyStream::new(body).map_ok(|frame| {
frame.into_data().unwrap_or_default()
}).map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)),
http_body_util::BodyStream::new(body)
.map_ok(|frame| frame.into_data().unwrap_or_default())
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)),
);
Box::pin(stream)
};
match state.storage.upload_part(bucket, upload_id, part_number, boxed).await {
match state
.storage
.upload_part(bucket, upload_id, part_number, boxed)
.await
{
Ok(etag) => {
let mut headers = HeaderMap::new();
headers.insert("etag", format!("\"{}\"", etag).parse().unwrap());
@@ -1052,7 +1112,11 @@ async fn complete_multipart_handler(
})
.collect();
match state.storage.complete_multipart(bucket, upload_id, &parts).await {
match state
.storage
.complete_multipart(bucket, upload_id, &parts)
.await
{
Ok(meta) => {
let etag = meta.etag.as_deref().unwrap_or("");
let xml = myfsio_xml::response::complete_multipart_upload_xml(
@@ -1067,21 +1131,14 @@ async fn complete_multipart_handler(
}
}
async fn abort_multipart_handler(
state: &AppState,
bucket: &str,
upload_id: &str,
) -> Response {
async fn abort_multipart_handler(state: &AppState, bucket: &str, upload_id: &str) -> Response {
match state.storage.abort_multipart(bucket, upload_id).await {
Ok(()) => StatusCode::NO_CONTENT.into_response(),
Err(e) => storage_err_response(e),
}
}
async fn list_multipart_uploads_handler(
state: &AppState,
bucket: &str,
) -> Response {
async fn list_multipart_uploads_handler(state: &AppState, bucket: &str) -> Response {
match state.storage.list_multipart_uploads(bucket).await {
Ok(uploads) => {
let xml = myfsio_xml::response::list_multipart_uploads_xml(bucket, &uploads);
@@ -1128,9 +1185,7 @@ async fn object_attributes_handler(
.collect();
let all = attrs.is_empty();
let mut xml = String::from(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
);
let mut xml = String::from("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
xml.push_str("<GetObjectAttributesResponse xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">");
if all || attrs.contains("etag") {
@@ -1139,10 +1194,7 @@ async fn object_attributes_handler(
}
}
if all || attrs.contains("storageclass") {
let sc = meta
.storage_class
.as_deref()
.unwrap_or("STANDARD");
let sc = meta.storage_class.as_deref().unwrap_or("STANDARD");
xml.push_str(&format!("<StorageClass>{}</StorageClass>", xml_escape(sc)));
}
if all || attrs.contains("objectsize") {
@@ -1185,7 +1237,11 @@ async fn copy_object_handler(
return resp;
}
match state.storage.copy_object(src_bucket, src_key, dst_bucket, dst_key).await {
match state
.storage
.copy_object(src_bucket, src_key, dst_bucket, dst_key)
.await
{
Ok(meta) => {
let etag = meta.etag.as_deref().unwrap_or("");
let last_modified = myfsio_xml::response::format_s3_datetime(&meta.last_modified);
@@ -1196,11 +1252,7 @@ async fn copy_object_handler(
}
}
async fn delete_objects_handler(
state: &AppState,
bucket: &str,
body: Body,
) -> Response {
async fn delete_objects_handler(state: &AppState, bucket: &str, body: Body) -> Response {
let body_bytes = match http_body_util::BodyExt::collect(body).await {
Ok(collected) => collected.to_bytes(),
Err(_) => {
@@ -1289,7 +1341,9 @@ async fn range_get_handler(
headers.insert("content-length", length.to_string().parse().unwrap());
headers.insert(
"content-range",
format!("bytes {}-{}/{}", start, end, total_size).parse().unwrap(),
format!("bytes {}-{}/{}", start, end, total_size)
.parse()
.unwrap(),
);
if let Some(ref etag) = meta.etag {
headers.insert("etag", format!("\"{}\"", etag).parse().unwrap());
@@ -1471,7 +1525,10 @@ async fn resolve_encryption_context(
bucket: &str,
headers: &HeaderMap,
) -> Option<myfsio_crypto::encryption::EncryptionContext> {
if let Some(alg) = headers.get("x-amz-server-side-encryption").and_then(|v| v.to_str().ok()) {
if let Some(alg) = headers
.get("x-amz-server-side-encryption")
.and_then(|v| v.to_str().ok())
{
let algorithm = match alg {
"AES256" => myfsio_crypto::encryption::SseAlgorithm::Aes256,
"aws:kms" => myfsio_crypto::encryption::SseAlgorithm::AwsKms,
@@ -1606,11 +1663,21 @@ async fn post_object_form_handler(
let key_template = match fields.get("key").cloned() {
Some(k) => k,
None => return s3_error_response(S3Error::new(S3ErrorCode::InvalidArgument, "Missing key field")),
None => {
return s3_error_response(S3Error::new(
S3ErrorCode::InvalidArgument,
"Missing key field",
))
}
};
let policy_b64 = match fields.get("policy").cloned() {
Some(v) => v,
None => return s3_error_response(S3Error::new(S3ErrorCode::InvalidArgument, "Missing policy field")),
None => {
return s3_error_response(S3Error::new(
S3ErrorCode::InvalidArgument,
"Missing policy field",
))
}
};
let signature = match fields
.iter()
@@ -1618,7 +1685,12 @@ async fn post_object_form_handler(
.map(|(_, v)| v.clone())
{
Some(v) => v,
None => return s3_error_response(S3Error::new(S3ErrorCode::InvalidArgument, "Missing signature")),
None => {
return s3_error_response(S3Error::new(
S3ErrorCode::InvalidArgument,
"Missing signature",
))
}
};
let credential = match fields
.iter()
@@ -1626,7 +1698,12 @@ async fn post_object_form_handler(
.map(|(_, v)| v.clone())
{
Some(v) => v,
None => return s3_error_response(S3Error::new(S3ErrorCode::InvalidArgument, "Missing credential")),
None => {
return s3_error_response(S3Error::new(
S3ErrorCode::InvalidArgument,
"Missing credential",
))
}
};
let algorithm = match fields
.iter()
@@ -1634,7 +1711,12 @@ async fn post_object_form_handler(
.map(|(_, v)| v.clone())
{
Some(v) => v,
None => return s3_error_response(S3Error::new(S3ErrorCode::InvalidArgument, "Missing algorithm")),
None => {
return s3_error_response(S3Error::new(
S3ErrorCode::InvalidArgument,
"Missing algorithm",
))
}
};
if algorithm != "AWS4-HMAC-SHA256" {
return s3_error_response(S3Error::new(
@@ -1667,7 +1749,10 @@ async fn post_object_form_handler(
match chrono::DateTime::parse_from_rfc3339(&normalized) {
Ok(exp_time) => {
if Utc::now() > exp_time.with_timezone(&Utc) {
return s3_error_response(S3Error::new(S3ErrorCode::AccessDenied, "Policy expired"));
return s3_error_response(S3Error::new(
S3ErrorCode::AccessDenied,
"Policy expired",
));
}
}
Err(_) => {
@@ -1688,14 +1773,23 @@ async fn post_object_form_handler(
};
if let Some(conditions) = policy_value.get("conditions").and_then(|v| v.as_array()) {
if let Err(msg) = validate_post_policy_conditions(bucket, &object_key, conditions, &fields, content_length) {
if let Err(msg) = validate_post_policy_conditions(
bucket,
&object_key,
conditions,
&fields,
content_length,
) {
return s3_error_response(S3Error::new(S3ErrorCode::AccessDenied, msg));
}
}
let credential_parts: Vec<&str> = credential.split('/').collect();
if credential_parts.len() != 5 {
return s3_error_response(S3Error::new(S3ErrorCode::InvalidArgument, "Invalid credential format"));
return s3_error_response(S3Error::new(
S3ErrorCode::InvalidArgument,
"Invalid credential format",
));
}
let access_key = credential_parts[0];
let date_stamp = credential_parts[1];
@@ -1704,9 +1798,15 @@ async fn post_object_form_handler(
let secret_key = match state.iam.get_secret_key(access_key) {
Some(s) => s,
None => return s3_error_response(S3Error::new(S3ErrorCode::AccessDenied, "Invalid access key")),
None => {
return s3_error_response(S3Error::new(
S3ErrorCode::AccessDenied,
"Invalid access key",
))
}
};
let signing_key = myfsio_auth::sigv4::derive_signing_key(&secret_key, date_stamp, region, service);
let signing_key =
myfsio_auth::sigv4::derive_signing_key(&secret_key, date_stamp, region, service);
let expected = myfsio_auth::sigv4::compute_post_policy_signature(&signing_key, &policy_b64);
if !myfsio_auth::sigv4::constant_time_compare(&expected, &signature) {
return s3_error_response(S3Error::new(
@@ -1717,7 +1817,12 @@ async fn post_object_form_handler(
let file_data = match file_bytes {
Some(b) => b,
None => return s3_error_response(S3Error::new(S3ErrorCode::InvalidArgument, "Missing file field")),
None => {
return s3_error_response(S3Error::new(
S3ErrorCode::InvalidArgument,
"Missing file field",
))
}
};
let mut metadata = HashMap::new();
@@ -1741,7 +1846,11 @@ async fn post_object_form_handler(
let cursor = std::io::Cursor::new(file_data.to_vec());
let boxed: myfsio_storage::traits::AsyncReadStream = Box::pin(cursor);
let meta = match state.storage.put_object(bucket, &object_key, boxed, Some(metadata)).await {
let meta = match state
.storage
.put_object(bucket, &object_key, boxed, Some(metadata))
.await
{
Ok(m) => m,
Err(e) => return storage_err_response(e),
};

94
rust/myfsio-engine/crates/myfsio-server/src/handlers/select.rs
View File

@@ -51,14 +51,12 @@ pub async fn post_select_object_content(
let object_path = match state.storage.get_object_path(bucket, key).await {
Ok(path) => path,
Err(_) => {
return s3_error_response(S3Error::new(
S3ErrorCode::NoSuchKey,
"Object not found",
));
return s3_error_response(S3Error::new(S3ErrorCode::NoSuchKey, "Object not found"));
}
};
let join_res = tokio::task::spawn_blocking(move || execute_select_query(object_path, request)).await;
let join_res =
tokio::task::spawn_blocking(move || execute_select_query(object_path, request)).await;
let chunks = match join_res {
Ok(Ok(chunks)) => chunks,
Ok(Err(message)) => {
@@ -79,7 +77,10 @@ pub async fn post_select_object_content(
}
let stats_payload = build_stats_xml(0, bytes_returned);
events.push(Bytes::from(encode_select_event("Stats", stats_payload.as_bytes())));
events.push(Bytes::from(encode_select_event(
"Stats",
stats_payload.as_bytes(),
)));
events.push(Bytes::from(encode_select_event("End", b"")));
let stream = stream::iter(events.into_iter().map(Ok::<Bytes, std::io::Error>));
@@ -166,10 +167,18 @@ fn parse_select_request(payload: &[u8]) -> Result<SelectRequest, S3Error> {
));
}
let input_node = child(&root, "InputSerialization")
.ok_or_else(|| S3Error::new(S3ErrorCode::InvalidRequest, "InputSerialization is required"))?;
let output_node = child(&root, "OutputSerialization")
.ok_or_else(|| S3Error::new(S3ErrorCode::InvalidRequest, "OutputSerialization is required"))?;
let input_node = child(&root, "InputSerialization").ok_or_else(|| {
S3Error::new(
S3ErrorCode::InvalidRequest,
"InputSerialization is required",
)
})?;
let output_node = child(&root, "OutputSerialization").ok_or_else(|| {
S3Error::new(
S3ErrorCode::InvalidRequest,
"OutputSerialization is required",
)
})?;
let input_format = parse_input_format(&input_node)?;
let output_format = parse_output_format(&output_node)?;
@@ -187,8 +196,10 @@ fn parse_input_format(node: &roxmltree::Node<'_, '_>) -> Result<InputFormat, S3E
file_header_info: child_text(&csv_node, "FileHeaderInfo")
.unwrap_or_else(|| "NONE".to_string())
.to_ascii_uppercase(),
field_delimiter: child_text(&csv_node, "FieldDelimiter").unwrap_or_else(|| ",".to_string()),
quote_character: child_text(&csv_node, "QuoteCharacter").unwrap_or_else(|| "\"".to_string()),
field_delimiter: child_text(&csv_node, "FieldDelimiter")
.unwrap_or_else(|| ",".to_string()),
quote_character: child_text(&csv_node, "QuoteCharacter")
.unwrap_or_else(|| "\"".to_string()),
}));
}
@@ -213,15 +224,19 @@ fn parse_input_format(node: &roxmltree::Node<'_, '_>) -> Result<InputFormat, S3E
fn parse_output_format(node: &roxmltree::Node<'_, '_>) -> Result<OutputFormat, S3Error> {
if let Some(csv_node) = child(node, "CSV") {
return Ok(OutputFormat::Csv(CsvOutputConfig {
field_delimiter: child_text(&csv_node, "FieldDelimiter").unwrap_or_else(|| ",".to_string()),
record_delimiter: child_text(&csv_node, "RecordDelimiter").unwrap_or_else(|| "\n".to_string()),
quote_character: child_text(&csv_node, "QuoteCharacter").unwrap_or_else(|| "\"".to_string()),
field_delimiter: child_text(&csv_node, "FieldDelimiter")
.unwrap_or_else(|| ",".to_string()),
record_delimiter: child_text(&csv_node, "RecordDelimiter")
.unwrap_or_else(|| "\n".to_string()),
quote_character: child_text(&csv_node, "QuoteCharacter")
.unwrap_or_else(|| "\"".to_string()),
}));
}
if let Some(json_node) = child(node, "JSON") {
return Ok(OutputFormat::Json(JsonOutputConfig {
record_delimiter: child_text(&json_node, "RecordDelimiter").unwrap_or_else(|| "\n".to_string()),
record_delimiter: child_text(&json_node, "RecordDelimiter")
.unwrap_or_else(|| "\n".to_string()),
}));
}
@@ -231,7 +246,10 @@ fn parse_output_format(node: &roxmltree::Node<'_, '_>) -> Result<OutputFormat, S
))
}
fn child<'a, 'input>(node: &'a roxmltree::Node<'a, 'input>, name: &str) -> Option<roxmltree::Node<'a, 'input>> {
fn child<'a, 'input>(
node: &'a roxmltree::Node<'a, 'input>,
name: &str,
) -> Option<roxmltree::Node<'a, 'input>> {
node.children()
.find(|n| n.is_element() && n.tag_name().name() == name)
}
@@ -243,7 +261,8 @@ fn child_text(node: &roxmltree::Node<'_, '_>, name: &str) -> Option<String> {
}
fn execute_select_query(path: PathBuf, request: SelectRequest) -> Result<Vec<Vec<u8>>, String> {
let conn = Connection::open_in_memory().map_err(|e| format!("DuckDB connection error: {}", e))?;
let conn =
Connection::open_in_memory().map_err(|e| format!("DuckDB connection error: {}", e))?;
load_input_table(&conn, &path, &request.input_format)?;
@@ -341,7 +360,10 @@ fn collect_csv_chunks(
let mut chunks: Vec<Vec<u8>> = Vec::new();
let mut buffer = String::new();
while let Some(row) = rows.next().map_err(|e| format!("SQL execution error: {}", e))? {
while let Some(row) = rows
.next()
.map_err(|e| format!("SQL execution error: {}", e))?
{
let mut fields: Vec<String> = Vec::with_capacity(col_count);
for i in 0..col_count {
let value = row
@@ -353,7 +375,10 @@ fn collect_csv_chunks(
}
let mut text = value_ref_to_string(value);
if text.contains(&delimiter) || text.contains(&quote) || text.contains(&record_delimiter) {
if text.contains(&delimiter)
|| text.contains(&quote)
|| text.contains(&record_delimiter)
{
text = text.replace(&quote, &(quote.clone() + &quote));
text = format!("{}{}{}", quote, text, quote);
}
@@ -385,16 +410,16 @@ fn collect_json_chunks(
let mut chunks: Vec<Vec<u8>> = Vec::new();
let mut buffer = String::new();
while let Some(row) = rows.next().map_err(|e| format!("SQL execution error: {}", e))? {
while let Some(row) = rows
.next()
.map_err(|e| format!("SQL execution error: {}", e))?
{
let mut record: HashMap<String, serde_json::Value> = HashMap::with_capacity(col_count);
for i in 0..col_count {
let value = row
.get_ref(i)
.map_err(|e| format!("SQL execution error: {}", e))?;
let key = columns
.get(i)
.cloned()
.unwrap_or_else(|| format!("_{}", i));
let key = columns.get(i).cloned().unwrap_or_else(|| format!("_{}", i));
record.insert(key, value_ref_to_json(value));
}
let line = serde_json::to_string(&record)
@@ -452,7 +477,9 @@ fn value_ref_to_json(value: ValueRef<'_>) -> serde_json::Value {
ValueRef::Double(v) => serde_json::json!(v),
ValueRef::Decimal(v) => serde_json::Value::String(v.to_string()),
ValueRef::Text(v) => serde_json::Value::String(String::from_utf8_lossy(v).into_owned()),
ValueRef::Blob(v) => serde_json::Value::String(base64::engine::general_purpose::STANDARD.encode(v)),
ValueRef::Blob(v) => {
serde_json::Value::String(base64::engine::general_purpose::STANDARD.encode(v))
}
_ => serde_json::Value::String(format!("{:?}", value)),
}
}
@@ -477,7 +504,8 @@ fn require_xml_content_type(headers: &HeaderMap) -> Option<Response> {
}
fn s3_error_response(err: S3Error) -> Response {
let status = StatusCode::from_u16(err.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
let status =
StatusCode::from_u16(err.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
let resource = if err.resource.is_empty() {
"/".to_string()
} else {
@@ -487,12 +515,7 @@ fn s3_error_response(err: S3Error) -> Response {
.with_resource(resource)
.with_request_id(uuid::Uuid::new_v4().simple().to_string())
.to_xml();
(
status,
[("content-type", "application/xml")],
body,
)
.into_response()
(status, [("content-type", "application/xml")], body).into_response()
}
fn build_stats_xml(bytes_scanned: usize, bytes_returned: usize) -> String {
@@ -508,7 +531,10 @@ fn encode_select_event(event_type: &str, payload: &[u8]) -> Vec<u8> {
let mut headers = Vec::new();
headers.extend(encode_select_header(":event-type", event_type));
if event_type == "Records" {
headers.extend(encode_select_header(":content-type", "application/octet-stream"));
headers.extend(encode_select_header(
":content-type",
"application/octet-stream",
));
} else if event_type == "Stats" {
headers.extend(encode_select_header(":content-type", "text/xml"));
}

56
rust/myfsio-engine/crates/myfsio-server/src/handlers/ui.rs
View File

@@ -1,4 +1,5 @@
use std::collections::HashMap;
use std::error::Error as StdError;
use axum::extract::{Extension, Form, State};
use axum::http::{header, HeaderMap, StatusCode};
@@ -100,6 +101,10 @@ pub async fn csrf_error_page(
resp
}
pub async fn root_redirect() -> Response {
Redirect::to("/ui/buckets").into_response()
}
pub async fn not_found_page(
State(state): State<AppState>,
Extension(session): Extension<SessionHandle>,
@@ -119,9 +124,15 @@ pub async fn require_login(
return next.run(req).await;
}
let path = req.uri().path().to_string();
let query = req.uri().query().map(|q| format!("?{}", q)).unwrap_or_default();
let query = req
.uri()
.query()
.map(|q| format!("?{}", q))
.unwrap_or_default();
let next_url = format!("{}{}", path, query);
let encoded = percent_encoding::utf8_percent_encode(&next_url, percent_encoding::NON_ALPHANUMERIC).to_string();
let encoded =
percent_encoding::utf8_percent_encode(&next_url, percent_encoding::NON_ALPHANUMERIC)
.to_string();
let target = format!("/login?next={}", encoded);
Redirect::to(&target).into_response()
}
@@ -130,22 +141,45 @@ pub fn render(state: &AppState, template: &str, ctx: &Context) -> Response {
let engine = match &state.templates {
Some(e) => e,
None => {
return (StatusCode::INTERNAL_SERVER_ERROR, "Templates not configured").into_response();
return (
StatusCode::INTERNAL_SERVER_ERROR,
"Templates not configured",
)
.into_response();
}
};
match engine.render(template, ctx) {
Ok(html) => {
let mut headers = HeaderMap::new();
headers.insert(header::CONTENT_TYPE, "text/html; charset=utf-8".parse().unwrap());
headers.insert(
header::CONTENT_TYPE,
"text/html; charset=utf-8".parse().unwrap(),
);
(StatusCode::OK, headers, html).into_response()
}
Err(e) => {
tracing::error!("Template render failed ({}): {}", template, e);
(
StatusCode::INTERNAL_SERVER_ERROR,
format!("Template error: {}", e),
)
.into_response()
let mut detail = format!("{}", e);
let mut src = StdError::source(&e);
while let Some(s) = src {
detail.push_str(" | ");
detail.push_str(&s.to_string());
src = s.source();
}
tracing::error!("Template render failed ({}): {}", template, detail);
let fallback_ctx = Context::new();
let body = if template != "500.html" {
engine
.render("500.html", &fallback_ctx)
.unwrap_or_else(|_| "Internal Server Error".to_string())
} else {
"Internal Server Error".to_string()
};
let mut headers = HeaderMap::new();
headers.insert(
header::CONTENT_TYPE,
"text/html; charset=utf-8".parse().unwrap(),
);
(StatusCode::INTERNAL_SERVER_ERROR, headers, body).into_response()
}
}
}
@@ -159,6 +193,8 @@ pub fn base_context(session: &SessionHandle, endpoint: Option<&str>) -> Context
ctx.insert("current_user_display_name", &snapshot.display_name);
ctx.insert("current_endpoint", &endpoint.unwrap_or(""));
ctx.insert("request_args", &HashMap::<String, String>::new());
ctx.insert("null", &serde_json::Value::Null);
ctx.insert("none", &serde_json::Value::Null);
ctx
}

3506
rust/myfsio-engine/crates/myfsio-server/src/handlers/ui_api.rs Normal file
View File

File diff suppressed because it is too large Load Diff

2899
rust/myfsio-engine/crates/myfsio-server/src/handlers/ui_pages.rs
View File

File diff suppressed because it is too large Load Diff

462
rust/myfsio-engine/crates/myfsio-server/src/lib.rs
View File

@@ -9,24 +9,249 @@ pub mod templates;
use axum::Router;
pub const SERVER_HEADER: &str = concat!("MyFSIO-Rust/", env!("CARGO_PKG_VERSION"));
pub const SERVER_HEADER: &str = "MyFSIO";
pub fn create_ui_router(state: state::AppState) -> Router {
use axum::routing::{get, post};
use axum::routing::{delete, get, post, put};
use handlers::ui;
use handlers::ui_api;
use handlers::ui_pages;
let protected = Router::new()
.route("/", get(ui::root_redirect))
.route("/ui", get(ui::root_redirect))
.route("/ui/", get(ui::root_redirect))
.route("/ui/buckets", get(ui_pages::buckets_overview))
.route("/ui/buckets/create", post(ui_pages::create_bucket))
.route("/ui/buckets/{bucket_name}", get(ui_pages::bucket_detail))
.route(
"/ui/buckets/{bucket_name}/delete",
post(ui_pages::delete_bucket),
)
.route(
"/ui/buckets/{bucket_name}/versioning",
post(ui_pages::update_bucket_versioning),
)
.route(
"/ui/buckets/{bucket_name}/quota",
post(ui_pages::update_bucket_quota),
)
.route(
"/ui/buckets/{bucket_name}/encryption",
post(ui_pages::update_bucket_encryption),
)
.route(
"/ui/buckets/{bucket_name}/policy",
post(ui_pages::update_bucket_policy),
)
.route(
"/ui/buckets/{bucket_name}/replication",
post(ui_pages::update_bucket_replication),
)
.route(
"/ui/buckets/{bucket_name}/website",
post(ui_pages::update_bucket_website),
)
.route(
"/ui/buckets/{bucket_name}/upload",
post(ui_api::upload_object),
)
.route(
"/ui/buckets/{bucket_name}/multipart/initiate",
post(ui_api::initiate_multipart_upload),
)
.route(
"/ui/buckets/{bucket_name}/multipart/{upload_id}/part",
put(ui_api::upload_multipart_part),
)
.route(
"/ui/buckets/{bucket_name}/multipart/{upload_id}/complete",
post(ui_api::complete_multipart_upload),
)
.route(
"/ui/buckets/{bucket_name}/multipart/{upload_id}/abort",
delete(ui_api::abort_multipart_upload),
)
.route(
"/ui/buckets/{bucket_name}/objects",
get(ui_api::list_bucket_objects),
)
.route(
"/ui/buckets/{bucket_name}/objects/stream",
get(ui_api::stream_bucket_objects),
)
.route(
"/ui/buckets/{bucket_name}/folders",
get(ui_api::list_bucket_folders),
)
.route(
"/ui/buckets/{bucket_name}/copy-targets",
get(ui_api::list_copy_targets),
)
.route(
"/ui/buckets/{bucket_name}/objects/{*rest}",
get(ui_api::object_get_dispatch).post(ui_api::object_post_dispatch),
)
.route(
"/ui/buckets/{bucket_name}/acl",
get(ui_api::bucket_acl).post(ui_api::update_bucket_acl),
)
.route(
"/ui/buckets/{bucket_name}/cors",
get(ui_api::bucket_cors).post(ui_api::update_bucket_cors),
)
.route(
"/ui/buckets/{bucket_name}/lifecycle",
get(ui_api::bucket_lifecycle).post(ui_api::update_bucket_lifecycle),
)
.route(
"/ui/buckets/{bucket_name}/lifecycle/history",
get(ui_api::lifecycle_history_stub),
)
.route(
"/ui/buckets/{bucket_name}/replication/status",
get(ui_api::replication_status),
)
.route(
"/ui/buckets/{bucket_name}/replication/failures",
get(ui_api::replication_failures).delete(ui_api::clear_replication_failures),
)
.route(
"/ui/buckets/{bucket_name}/replication/failures/retry",
post(ui_api::retry_replication_failure),
)
.route(
"/ui/buckets/{bucket_name}/replication/failures/retry-all",
post(ui_api::retry_all_replication_failures),
)
.route(
"/ui/buckets/{bucket_name}/replication/failures/dismiss",
delete(ui_api::dismiss_replication_failure),
)
.route(
"/ui/buckets/{bucket_name}/replication/failures/clear",
delete(ui_api::clear_replication_failures),
)
.route(
"/ui/buckets/{bucket_name}/bulk-delete",
post(ui_api::bulk_delete_objects),
)
.route(
"/ui/buckets/{bucket_name}/bulk-download",
post(ui_api::bulk_download_objects),
)
.route(
"/ui/buckets/{bucket_name}/archived",
get(ui_api::archived_objects),
)
.route(
"/ui/buckets/{bucket_name}/archived/{*rest}",
post(ui_api::archived_post_dispatch),
)
.route("/ui/iam", get(ui_pages::iam_dashboard))
.route("/ui/iam/users", post(ui_pages::create_iam_user))
.route("/ui/iam/users/{user_id}", post(ui_pages::update_iam_user))
.route(
"/ui/iam/users/{user_id}/delete",
post(ui_pages::delete_iam_user),
)
.route(
"/ui/iam/users/{user_id}/policies",
post(ui_pages::update_iam_policies),
)
.route(
"/ui/iam/users/{user_id}/expiry",
post(ui_pages::update_iam_expiry),
)
.route(
"/ui/iam/users/{user_id}/rotate-secret",
post(ui_pages::rotate_iam_secret),
)
.route("/ui/connections/create", post(ui_pages::create_connection))
.route("/ui/connections/test", post(ui_api::test_connection))
.route(
"/ui/connections/{connection_id}",
post(ui_pages::update_connection),
)
.route(
"/ui/connections/{connection_id}/delete",
post(ui_pages::delete_connection),
)
.route(
"/ui/connections/{connection_id}/health",
get(ui_api::connection_health),
)
.route("/ui/sites", get(ui_pages::sites_dashboard))
.route("/ui/sites/local", post(ui_pages::update_local_site))
.route("/ui/sites/peers", post(ui_pages::add_peer_site))
.route(
"/ui/sites/peers/{site_id}/update",
post(ui_pages::update_peer_site),
)
.route(
"/ui/sites/peers/{site_id}/delete",
post(ui_pages::delete_peer_site),
)
.route("/ui/sites/peers/{site_id}/health", get(ui_api::peer_health))
.route(
"/ui/sites/peers/{site_id}/sync-stats",
get(ui_api::peer_sync_stats),
)
.route(
"/ui/sites/peers/{site_id}/bidirectional-status",
get(ui_api::peer_bidirectional_status),
)
.route("/ui/connections", get(ui_pages::connections_dashboard))
.route("/ui/metrics", get(ui_pages::metrics_dashboard))
.route(
"/ui/metrics/settings",
get(ui_api::metrics_settings).put(ui_api::update_metrics_settings),
)
.route("/ui/metrics/api", get(ui_api::metrics_api))
.route("/ui/metrics/history", get(ui_api::metrics_history))
.route("/ui/metrics/operations", get(ui_api::metrics_operations))
.route(
"/ui/metrics/operations/history",
get(ui_api::metrics_operations_history),
)
.route("/ui/system", get(ui_pages::system_dashboard))
.route("/ui/website-domains", get(ui_pages::website_domains_dashboard))
.route("/ui/system/gc/status", get(ui_api::gc_status_ui))
.route("/ui/system/gc/run", post(ui_api::gc_run_ui))
.route("/ui/system/gc/history", get(ui_api::gc_history_ui))
.route(
"/ui/system/integrity/status",
get(ui_api::integrity_status_ui),
)
.route("/ui/system/integrity/run", post(ui_api::integrity_run_ui))
.route(
"/ui/system/integrity/history",
get(ui_api::integrity_history_ui),
)
.route(
"/ui/website-domains",
get(ui_pages::website_domains_dashboard),
)
.route(
"/ui/website-domains/create",
post(ui_pages::create_website_domain),
)
.route(
"/ui/website-domains/{domain}",
post(ui_pages::update_website_domain),
)
.route(
"/ui/website-domains/{domain}/delete",
post(ui_pages::delete_website_domain),
)
.route("/ui/replication/new", get(ui_pages::replication_wizard))
.route(
"/ui/replication/create",
post(ui_pages::create_peer_replication_rules_from_query),
)
.route(
"/ui/sites/peers/{site_id}/replication-rules",
post(ui_pages::create_peer_replication_rules),
)
.route("/ui/docs", get(ui_pages::docs_page))
.layer(axum::middleware::from_fn(ui::require_login));
@@ -40,18 +265,29 @@ pub fn create_ui_router(state: state::AppState) -> Router {
secure: false,
};
let static_service = tower_http::services::ServeDir::new(&state.config.static_dir);
protected
.merge(public)
.fallback(ui::not_found_page)
.layer(axum::middleware::from_fn(middleware::csrf_layer))
.layer(axum::middleware::from_fn_with_state(
session_state,
middleware::session_layer,
))
.layer(axum::middleware::from_fn_with_state(
state.clone(),
middleware::ui_metrics_layer,
))
.with_state(state)
.nest_service("/static", static_service)
.layer(axum::middleware::from_fn(middleware::server_header))
.layer(tower_http::compression::CompressionLayer::new())
}
pub fn create_router(state: state::AppState) -> Router {
let mut router = Router::new()
.route("/myfsio/health", axum::routing::get(handlers::health_check))
.route("/", axum::routing::get(handlers::list_buckets))
.route(
"/{bucket}",
@@ -61,6 +297,14 @@ pub fn create_router(state: state::AppState) -> Router {
.head(handlers::head_bucket)
.post(handlers::post_bucket),
)
.route(
"/{bucket}/",
axum::routing::put(handlers::create_bucket)
.get(handlers::get_bucket)
.delete(handlers::delete_bucket)
.head(handlers::head_bucket)
.post(handlers::post_bucket),
)
.route(
"/{bucket}/{*key}",
axum::routing::put(handlers::put_object)
@@ -72,53 +316,189 @@ pub fn create_router(state: state::AppState) -> Router {
if state.config.kms_enabled {
router = router
.route("/kms/keys", axum::routing::get(handlers::kms::list_keys).post(handlers::kms::create_key))
.route("/kms/keys/{key_id}", axum::routing::get(handlers::kms::get_key).delete(handlers::kms::delete_key))
.route("/kms/keys/{key_id}/enable", axum::routing::post(handlers::kms::enable_key))
.route("/kms/keys/{key_id}/disable", axum::routing::post(handlers::kms::disable_key))
.route(
"/kms/keys",
axum::routing::get(handlers::kms::list_keys).post(handlers::kms::create_key),
)
.route(
"/kms/keys/{key_id}",
axum::routing::get(handlers::kms::get_key).delete(handlers::kms::delete_key),
)
.route(
"/kms/keys/{key_id}/enable",
axum::routing::post(handlers::kms::enable_key),
)
.route(
"/kms/keys/{key_id}/disable",
axum::routing::post(handlers::kms::disable_key),
)
.route("/kms/encrypt", axum::routing::post(handlers::kms::encrypt))
.route("/kms/decrypt", axum::routing::post(handlers::kms::decrypt))
.route("/kms/generate-data-key", axum::routing::post(handlers::kms::generate_data_key));
.route(
"/kms/generate-data-key",
axum::routing::post(handlers::kms::generate_data_key),
)
.route(
"/kms/generate-data-key-without-plaintext",
axum::routing::post(handlers::kms::generate_data_key_without_plaintext),
)
.route(
"/kms/re-encrypt",
axum::routing::post(handlers::kms::re_encrypt),
)
.route(
"/kms/generate-random",
axum::routing::post(handlers::kms::generate_random),
)
.route(
"/kms/client/generate-key",
axum::routing::post(handlers::kms::client_generate_key),
)
.route(
"/kms/client/encrypt",
axum::routing::post(handlers::kms::client_encrypt),
)
.route(
"/kms/client/decrypt",
axum::routing::post(handlers::kms::client_decrypt),
)
.route(
"/kms/materials/{key_id}",
axum::routing::post(handlers::kms::materials),
);
}
router = router
.route("/admin/site/local", axum::routing::get(handlers::admin::get_local_site).put(handlers::admin::update_local_site))
.route("/admin/site/all", axum::routing::get(handlers::admin::list_all_sites))
.route("/admin/site/peers", axum::routing::post(handlers::admin::register_peer_site))
.route("/admin/site/peers/{site_id}", axum::routing::get(handlers::admin::get_peer_site).put(handlers::admin::update_peer_site).delete(handlers::admin::delete_peer_site))
.route("/admin/site/peers/{site_id}/health", axum::routing::post(handlers::admin::check_peer_health))
.route("/admin/site/topology", axum::routing::get(handlers::admin::get_topology))
.route("/admin/site/peers/{site_id}/bidirectional-status", axum::routing::get(handlers::admin::check_bidirectional_status))
.route("/admin/iam/users", axum::routing::get(handlers::admin::iam_list_users))
.route("/admin/iam/users/{identifier}", axum::routing::get(handlers::admin::iam_get_user))
.route("/admin/iam/users/{identifier}/policies", axum::routing::get(handlers::admin::iam_get_user_policies))
.route("/admin/iam/users/{identifier}/access-keys", axum::routing::post(handlers::admin::iam_create_access_key))
.route("/admin/iam/users/{identifier}/access-keys/{access_key}", axum::routing::delete(handlers::admin::iam_delete_access_key))
.route("/admin/iam/users/{identifier}/disable", axum::routing::post(handlers::admin::iam_disable_user))
.route("/admin/iam/users/{identifier}/enable", axum::routing::post(handlers::admin::iam_enable_user))
.route("/admin/website-domains", axum::routing::get(handlers::admin::list_website_domains).post(handlers::admin::create_website_domain))
.route("/admin/website-domains/{domain}", axum::routing::get(handlers::admin::get_website_domain).put(handlers::admin::update_website_domain).delete(handlers::admin::delete_website_domain))
.route("/admin/gc/status", axum::routing::get(handlers::admin::gc_status))
.route("/admin/gc/run", axum::routing::post(handlers::admin::gc_run))
.route("/admin/gc/history", axum::routing::get(handlers::admin::gc_history))
.route("/admin/integrity/status", axum::routing::get(handlers::admin::integrity_status))
.route("/admin/integrity/run", axum::routing::post(handlers::admin::integrity_run))
.route("/admin/integrity/history", axum::routing::get(handlers::admin::integrity_history));
.route(
"/admin/site",
axum::routing::get(handlers::admin::get_local_site)
.put(handlers::admin::update_local_site),
)
.route(
"/admin/sites",
axum::routing::get(handlers::admin::list_all_sites)
.post(handlers::admin::register_peer_site),
)
.route(
"/admin/sites/{site_id}",
axum::routing::get(handlers::admin::get_peer_site)
.put(handlers::admin::update_peer_site)
.delete(handlers::admin::delete_peer_site),
)
.route(
"/admin/sites/{site_id}/health",
axum::routing::get(handlers::admin::check_peer_health)
.post(handlers::admin::check_peer_health),
)
.route(
"/admin/sites/{site_id}/bidirectional-status",
axum::routing::get(handlers::admin::check_bidirectional_status),
)
.route(
"/admin/topology",
axum::routing::get(handlers::admin::get_topology),
)
.route(
"/admin/site/local",
axum::routing::get(handlers::admin::get_local_site)
.put(handlers::admin::update_local_site),
)
.route(
"/admin/site/all",
axum::routing::get(handlers::admin::list_all_sites),
)
.route(
"/admin/site/peers",
axum::routing::post(handlers::admin::register_peer_site),
)
.route(
"/admin/site/peers/{site_id}",
axum::routing::get(handlers::admin::get_peer_site)
.put(handlers::admin::update_peer_site)
.delete(handlers::admin::delete_peer_site),
)
.route(
"/admin/site/peers/{site_id}/health",
axum::routing::post(handlers::admin::check_peer_health),
)
.route(
"/admin/site/topology",
axum::routing::get(handlers::admin::get_topology),
)
.route(
"/admin/site/peers/{site_id}/bidirectional-status",
axum::routing::get(handlers::admin::check_bidirectional_status),
)
.route(
"/admin/iam/users",
axum::routing::get(handlers::admin::iam_list_users),
)
.route(
"/admin/iam/users/{identifier}",
axum::routing::get(handlers::admin::iam_get_user),
)
.route(
"/admin/iam/users/{identifier}/policies",
axum::routing::get(handlers::admin::iam_get_user_policies),
)
.route(
"/admin/iam/users/{identifier}/access-keys",
axum::routing::post(handlers::admin::iam_create_access_key),
)
.route(
"/admin/iam/users/{identifier}/access-keys/{access_key}",
axum::routing::delete(handlers::admin::iam_delete_access_key),
)
.route(
"/admin/iam/users/{identifier}/disable",
axum::routing::post(handlers::admin::iam_disable_user),
)
.route(
"/admin/iam/users/{identifier}/enable",
axum::routing::post(handlers::admin::iam_enable_user),
)
.route(
"/admin/website-domains",
axum::routing::get(handlers::admin::list_website_domains)
.post(handlers::admin::create_website_domain),
)
.route(
"/admin/website-domains/{domain}",
axum::routing::get(handlers::admin::get_website_domain)
.put(handlers::admin::update_website_domain)
.delete(handlers::admin::delete_website_domain),
)
.route(
"/admin/gc/status",
axum::routing::get(handlers::admin::gc_status),
)
.route(
"/admin/gc/run",
axum::routing::post(handlers::admin::gc_run),
)
.route(
"/admin/gc/history",
axum::routing::get(handlers::admin::gc_history),
)
.route(
"/admin/integrity/status",
axum::routing::get(handlers::admin::integrity_status),
)
.route(
"/admin/integrity/run",
axum::routing::post(handlers::admin::integrity_run),
)
.route(
"/admin/integrity/history",
axum::routing::get(handlers::admin::integrity_history),
);
let mut router = router
router
.layer(axum::middleware::from_fn_with_state(
state.clone(),
middleware::auth_layer,
))
.layer(axum::middleware::from_fn(middleware::server_header))
.with_state(state.clone());
if state.config.ui_enabled {
let static_service = tower_http::services::ServeDir::new(&state.config.static_dir);
router = router
.nest_service("/static", static_service)
.merge(create_ui_router(state));
}
router
.layer(tower_http::compression::CompressionLayer::new())
.with_state(state)
}

348
rust/myfsio-engine/crates/myfsio-server/src/main.rs
View File

@@ -3,8 +3,18 @@ use myfsio_server::config::ServerConfig;
use myfsio_server::state::AppState;
#[derive(Parser)]
#[command(name = "myfsio", version, about = "MyFSIO S3-compatible storage engine")]
#[command(
name = "myfsio",
version,
about = "MyFSIO S3-compatible storage engine"
)]
struct Cli {
#[arg(long, help = "Validate configuration and exit")]
check_config: bool,
#[arg(long, help = "Show configuration summary and exit")]
show_config: bool,
#[arg(long, help = "Reset admin credentials and exit")]
reset_cred: bool,
#[command(subcommand)]
command: Option<Command>,
}
@@ -17,9 +27,30 @@ enum Command {
#[tokio::main]
async fn main() {
load_env_files();
tracing_subscriber::fmt::init();
let cli = Cli::parse();
let config = ServerConfig::from_env();
if cli.reset_cred {
reset_admin_credentials(&config);
return;
}
if cli.check_config || cli.show_config {
print_config_summary(&config);
if cli.check_config {
let issues = validate_config(&config);
for issue in &issues {
println!("{issue}");
}
if issues.iter().any(|issue| issue.starts_with("CRITICAL:")) {
std::process::exit(1);
}
}
return;
}
match cli.command.unwrap_or(Command::Serve) {
Command::Version => {
println!("myfsio {}", env!("CARGO_PKG_VERSION"));
@@ -28,19 +59,24 @@ async fn main() {
Command::Serve => {}
}
let config = ServerConfig::from_env();
ensure_iam_bootstrap(&config);
let bind_addr = config.bind_addr;
let ui_bind_addr = config.ui_bind_addr;
tracing::info!("MyFSIO Rust Engine starting on {}", bind_addr);
tracing::info!("MyFSIO Rust Engine starting — API on {}", bind_addr);
if config.ui_enabled {
tracing::info!("UI will bind on {}", ui_bind_addr);
}
tracing::info!("Storage root: {}", config.storage_root.display());
tracing::info!("Region: {}", config.region);
tracing::info!(
"Encryption: {}, KMS: {}, GC: {}, Lifecycle: {}, Integrity: {}, Metrics: {}, UI: {}",
"Encryption: {}, KMS: {}, GC: {}, Lifecycle: {}, Integrity: {}, Metrics History: {}, Operation Metrics: {}, UI: {}",
config.encryption_enabled,
config.kms_enabled,
config.gc_enabled,
config.lifecycle_enabled,
config.integrity_enabled,
config.metrics_history_enabled,
config.metrics_enabled,
config.ui_enabled
);
@@ -68,13 +104,17 @@ async fn main() {
tracing::info!("Metrics collector background service started");
}
if let Some(ref system_metrics) = state.system_metrics {
bg_handles.push(system_metrics.clone().start_background());
tracing::info!("System metrics history collector started");
}
if config.lifecycle_enabled {
let lifecycle = std::sync::Arc::new(
myfsio_server::services::lifecycle::LifecycleService::new(
let lifecycle =
std::sync::Arc::new(myfsio_server::services::lifecycle::LifecycleService::new(
state.storage.clone(),
myfsio_server::services::lifecycle::LifecycleConfig::default(),
),
);
));
bg_handles.push(lifecycle.start_background());
tracing::info!("Lifecycle manager background service started");
}
@@ -87,15 +127,21 @@ async fn main() {
tracing::info!("Site sync worker started");
}
let app = myfsio_server::create_router(state);
let ui_enabled = config.ui_enabled;
let api_app = myfsio_server::create_router(state.clone());
let ui_app = if ui_enabled {
Some(myfsio_server::create_ui_router(state.clone()))
} else {
None
};
let listener = match tokio::net::TcpListener::bind(bind_addr).await {
let api_listener = match tokio::net::TcpListener::bind(bind_addr).await {
Ok(listener) => listener,
Err(err) => {
if err.kind() == std::io::ErrorKind::AddrInUse {
tracing::error!("Port already in use: {}", bind_addr);
tracing::error!("API port already in use: {}", bind_addr);
} else {
tracing::error!("Failed to bind {}: {}", bind_addr, err);
tracing::error!("Failed to bind API {}: {}", bind_addr, err);
}
for handle in bg_handles {
handle.abort();
@@ -103,17 +149,67 @@ async fn main() {
std::process::exit(1);
}
};
tracing::info!("Listening on {}", bind_addr);
tracing::info!("API listening on {}", bind_addr);
if let Err(err) = axum::serve(listener, app)
.with_graceful_shutdown(shutdown_signal())
.await
{
tracing::error!("Server exited with error: {}", err);
for handle in bg_handles {
handle.abort();
let ui_listener = if let Some(ref app) = ui_app {
let _ = app;
match tokio::net::TcpListener::bind(ui_bind_addr).await {
Ok(listener) => {
tracing::info!("UI listening on {}", ui_bind_addr);
Some(listener)
}
Err(err) => {
if err.kind() == std::io::ErrorKind::AddrInUse {
tracing::error!("UI port already in use: {}", ui_bind_addr);
} else {
tracing::error!("Failed to bind UI {}: {}", ui_bind_addr, err);
}
for handle in bg_handles {
handle.abort();
}
std::process::exit(1);
}
}
} else {
None
};
let shutdown = shutdown_signal_shared();
let api_shutdown = shutdown.clone();
let api_task = tokio::spawn(async move {
axum::serve(api_listener, api_app)
.with_graceful_shutdown(async move {
api_shutdown.notified().await;
})
.await
});
let ui_task = if let (Some(listener), Some(app)) = (ui_listener, ui_app) {
let ui_shutdown = shutdown.clone();
Some(tokio::spawn(async move {
axum::serve(listener, app)
.with_graceful_shutdown(async move {
ui_shutdown.notified().await;
})
.await
}))
} else {
None
};
tokio::signal::ctrl_c()
.await
.expect("Failed to listen for Ctrl+C");
tracing::info!("Shutdown signal received");
shutdown.notify_waiters();
if let Err(err) = api_task.await.unwrap_or(Ok(())) {
tracing::error!("API server exited with error: {}", err);
}
if let Some(task) = ui_task {
if let Err(err) = task.await.unwrap_or(Ok(())) {
tracing::error!("UI server exited with error: {}", err);
}
std::process::exit(1);
}
for handle in bg_handles {
@@ -121,9 +217,209 @@ async fn main() {
}
}
async fn shutdown_signal() {
tokio::signal::ctrl_c()
.await
.expect("Failed to listen for Ctrl+C");
tracing::info!("Shutdown signal received");
fn print_config_summary(config: &ServerConfig) {
println!("MyFSIO Rust Configuration");
println!("Version: {}", env!("CARGO_PKG_VERSION"));
println!("API bind: {}", config.bind_addr);
println!("UI bind: {}", config.ui_bind_addr);
println!("UI enabled: {}", config.ui_enabled);
println!("Storage root: {}", config.storage_root.display());
println!("IAM config: {}", config.iam_config_path.display());
println!("Region: {}", config.region);
println!("Encryption enabled: {}", config.encryption_enabled);
println!("KMS enabled: {}", config.kms_enabled);
println!("GC enabled: {}", config.gc_enabled);
println!("Integrity enabled: {}", config.integrity_enabled);
println!("Lifecycle enabled: {}", config.lifecycle_enabled);
println!(
"Website hosting enabled: {}",
config.website_hosting_enabled
);
println!("Site sync enabled: {}", config.site_sync_enabled);
println!(
"Metrics history enabled: {}",
config.metrics_history_enabled
);
println!("Operation metrics enabled: {}", config.metrics_enabled);
}
fn validate_config(config: &ServerConfig) -> Vec<String> {
let mut issues = Vec::new();
if config.ui_enabled && config.bind_addr == config.ui_bind_addr {
issues.push(
"CRITICAL: API and UI bind addresses cannot be identical when UI is enabled."
.to_string(),
);
}
if config.presigned_url_min_expiry > config.presigned_url_max_expiry {
issues.push("CRITICAL: PRESIGNED_URL_MIN_EXPIRY_SECONDS cannot exceed PRESIGNED_URL_MAX_EXPIRY_SECONDS.".to_string());
}
if let Err(err) = std::fs::create_dir_all(&config.storage_root) {
issues.push(format!(
"CRITICAL: Cannot create storage root {}: {}",
config.storage_root.display(),
err
));
}
if let Some(parent) = config.iam_config_path.parent() {
if let Err(err) = std::fs::create_dir_all(parent) {
issues.push(format!(
"CRITICAL: Cannot create IAM config directory {}: {}",
parent.display(),
err
));
}
}
if config.encryption_enabled && config.secret_key.is_none() {
issues.push(
"WARNING: ENCRYPTION_ENABLED=true but SECRET_KEY is not configured; secure-at-rest config encryption is unavailable.".to_string(),
);
}
if config.site_sync_enabled && !config.website_hosting_enabled {
issues.push(
"INFO: SITE_SYNC_ENABLED=true without WEBSITE_HOSTING_ENABLED; this is valid but unrelated.".to_string(),
);
}
issues
}
fn shutdown_signal_shared() -> std::sync::Arc<tokio::sync::Notify> {
std::sync::Arc::new(tokio::sync::Notify::new())
}
fn load_env_files() {
let cwd = std::env::current_dir().ok();
let mut candidates: Vec<std::path::PathBuf> = Vec::new();
candidates.push(std::path::PathBuf::from("/opt/myfsio/myfsio.env"));
if let Some(ref dir) = cwd {
candidates.push(dir.join(".env"));
candidates.push(dir.join("myfsio.env"));
for ancestor in dir.ancestors().skip(1).take(4) {
candidates.push(ancestor.join(".env"));
candidates.push(ancestor.join("myfsio.env"));
}
}
let mut seen = std::collections::HashSet::new();
for path in candidates {
if !seen.insert(path.clone()) {
continue;
}
if path.is_file() {
match dotenvy::from_path_override(&path) {
Ok(()) => eprintln!("Loaded env file: {}", path.display()),
Err(e) => eprintln!("Failed to load env file {}: {}", path.display(), e),
}
}
}
}
fn ensure_iam_bootstrap(config: &ServerConfig) {
let iam_path = &config.iam_config_path;
if iam_path.exists() {
return;
}
let access_key = std::env::var("ADMIN_ACCESS_KEY")
.ok()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.unwrap_or_else(|| format!("AK{}", uuid::Uuid::new_v4().simple()));
let secret_key = std::env::var("ADMIN_SECRET_KEY")
.ok()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.unwrap_or_else(|| format!("SK{}", uuid::Uuid::new_v4().simple()));
let user_id = format!("u-{}", &uuid::Uuid::new_v4().simple().to_string()[..16]);
let created_at = chrono::Utc::now().to_rfc3339();
let body = serde_json::json!({
"version": 2,
"users": [{
"user_id": user_id,
"display_name": "Local Admin",
"enabled": true,
"access_keys": [{
"access_key": access_key,
"secret_key": secret_key,
"status": "active",
"created_at": created_at,
}],
"policies": [{
"bucket": "*",
"actions": ["*"],
"prefix": "*",
}]
}]
});
let json = match serde_json::to_string_pretty(&body) {
Ok(s) => s,
Err(e) => {
tracing::error!("Failed to serialize IAM bootstrap config: {}", e);
return;
}
};
if let Some(parent) = iam_path.parent() {
if let Err(e) = std::fs::create_dir_all(parent) {
tracing::error!(
"Failed to create IAM config dir {}: {}",
parent.display(),
e
);
return;
}
}
if let Err(e) = std::fs::write(iam_path, json) {
tracing::error!(
"Failed to write IAM bootstrap config {}: {}",
iam_path.display(),
e
);
return;
}
tracing::info!("============================================================");
tracing::info!("MYFSIO - ADMIN CREDENTIALS INITIALIZED");
tracing::info!("============================================================");
tracing::info!("Access Key: {}", access_key);
tracing::info!("Secret Key: {}", secret_key);
tracing::info!("Saved to: {}", iam_path.display());
tracing::info!("============================================================");
}
fn reset_admin_credentials(config: &ServerConfig) {
if let Some(parent) = config.iam_config_path.parent() {
if let Err(err) = std::fs::create_dir_all(parent) {
eprintln!(
"Failed to create IAM config directory {}: {}",
parent.display(),
err
);
std::process::exit(1);
}
}
if config.iam_config_path.exists() {
let backup = config
.iam_config_path
.with_extension(format!("bak-{}", chrono::Utc::now().timestamp()));
if let Err(err) = std::fs::rename(&config.iam_config_path, &backup) {
eprintln!(
"Failed to back up existing IAM config {}: {}",
config.iam_config_path.display(),
err
);
std::process::exit(1);
}
println!("Backed up existing IAM config to {}", backup.display());
}
ensure_iam_bootstrap(config);
println!("Admin credentials reset.");
}

962
rust/myfsio-engine/crates/myfsio-server/src/middleware/auth.rs
View File

File diff suppressed because it is too large Load Diff

77
rust/myfsio-engine/crates/myfsio-server/src/middleware/mod.rs
View File

@@ -4,15 +4,82 @@ pub mod session;
pub use auth::auth_layer;
pub use session::{csrf_layer, session_layer, SessionHandle, SessionLayerState};
use axum::extract::Request;
use axum::extract::{Request, State};
use axum::middleware::Next;
use axum::response::Response;
use std::time::Instant;
use crate::state::AppState;
pub async fn server_header(req: Request, next: Next) -> Response {
let mut resp = next.run(req).await;
resp.headers_mut().insert(
"server",
crate::SERVER_HEADER.parse().unwrap(),
);
resp.headers_mut()
.insert("server", crate::SERVER_HEADER.parse().unwrap());
resp
}
pub async fn ui_metrics_layer(State(state): State<AppState>, req: Request, next: Next) -> Response {
let metrics = match state.metrics.clone() {
Some(m) => m,
None => return next.run(req).await,
};
let start = Instant::now();
let method = req.method().clone();
let path = req.uri().path().to_string();
let endpoint_type = classify_ui_endpoint(&path);
let bytes_in = req
.headers()
.get(axum::http::header::CONTENT_LENGTH)
.and_then(|v| v.to_str().ok())
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(0);
let response = next.run(req).await;
let latency_ms = start.elapsed().as_secs_f64() * 1000.0;
let status = response.status().as_u16();
let bytes_out = response
.headers()
.get(axum::http::header::CONTENT_LENGTH)
.and_then(|v| v.to_str().ok())
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(0);
let error_code = if status >= 400 { Some("UIError") } else { None };
metrics.record_request(
method.as_str(),
endpoint_type,
status,
latency_ms,
bytes_in,
bytes_out,
error_code,
);
response
}
fn classify_ui_endpoint(path: &str) -> &'static str {
if path.contains("/upload") {
"ui_upload"
} else if path.starts_with("/ui/buckets/") {
"ui_bucket"
} else if path.starts_with("/ui/iam") {
"ui_iam"
} else if path.starts_with("/ui/sites") {
"ui_sites"
} else if path.starts_with("/ui/connections") {
"ui_connections"
} else if path.starts_with("/ui/metrics") {
"ui_metrics"
} else if path.starts_with("/ui/system") {
"ui_system"
} else if path.starts_with("/ui/website-domains") {
"ui_website_domains"
} else if path.starts_with("/ui/replication") {
"ui_replication"
} else if path.starts_with("/login") || path.starts_with("/logout") {
"ui_auth"
} else {
"ui_other"
}
}

65
rust/myfsio-engine/crates/myfsio-server/src/middleware/session.rs
View File

@@ -62,18 +62,14 @@ pub async fn session_layer(
) -> Response {
let cookie_id = extract_session_cookie(&req);
let (session_id, session_data, is_new) = match cookie_id.and_then(|id| {
state
.store
.get(&id)
.map(|data| (id.clone(), data))
}) {
Some((id, data)) => (id, data, false),
None => {
let (id, data) = state.store.create();
(id, data, true)
}
};
let (session_id, session_data, is_new) =
match cookie_id.and_then(|id| state.store.get(&id).map(|data| (id.clone(), data))) {
Some((id, data)) => (id, data, false),
None => {
let (id, data) = state.store.create();
(id, data, true)
}
};
let handle = SessionHandle::new(session_id.clone(), session_data);
req.extensions_mut().insert(handle.clone());
@@ -95,6 +91,8 @@ pub async fn session_layer(
}
pub async fn csrf_layer(req: Request, next: Next) -> Response {
const CSRF_HEADER_ALIAS: &str = "x-csrftoken";
let method = req.method().clone();
let needs_check = matches!(
method,
@@ -126,29 +124,33 @@ pub async fn csrf_layer(req: Request, next: Next) -> Response {
let header_token = req
.headers()
.get(CSRF_HEADER_NAME)
.or_else(|| req.headers().get(CSRF_HEADER_ALIAS))
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string());
if let Some(token) = header_token {
if csrf_tokens_match(&expected, &token) {
if let Some(token) = header_token.as_deref() {
if csrf_tokens_match(&expected, token) {
return next.run(req).await;
}
}
let content_type = req
.headers()
.get(header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
let (parts, body) = req.into_parts();
let bytes = match axum::body::to_bytes(body, usize::MAX).await {
Ok(b) => b,
Err(_) => return (StatusCode::BAD_REQUEST, "Body read failed").into_response(),
};
let content_type = parts
.headers
.get(header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or("");
let form_token = if content_type.starts_with("application/x-www-form-urlencoded") {
extract_form_token(&bytes)
} else if content_type.starts_with("multipart/form-data") {
extract_multipart_token(&content_type, &bytes)
} else {
None
};
@@ -160,9 +162,32 @@ pub async fn csrf_layer(req: Request, next: Next) -> Response {
}
}
tracing::warn!(
path = %parts.uri.path(),
content_type = %content_type,
expected_len = expected.len(),
header_present = header_token.is_some(),
"CSRF token mismatch"
);
(StatusCode::FORBIDDEN, "Invalid CSRF token").into_response()
}
fn extract_multipart_token(content_type: &str, body: &[u8]) -> Option<String> {
let boundary = multer::parse_boundary(content_type).ok()?;
let prefix = format!("--{}", boundary);
let text = std::str::from_utf8(body).ok()?;
let needle = "name=\"csrf_token\"";
let idx = text.find(needle)?;
let after = &text[idx + needle.len()..];
let body_start = after.find("\r\n\r\n")? + 4;
let tail = &after[body_start..];
let end = tail
.find(&format!("\r\n--{}", prefix.trim_start_matches("--")))
.or_else(|| tail.find("\r\n--"))
.unwrap_or(tail.len());
Some(tail[..end].trim().to_string())
}
fn extract_session_cookie(req: &Request) -> Option<String> {
let raw = req.headers().get(header::COOKIE)?.to_str().ok()?;
for pair in raw.split(';') {

105
rust/myfsio-engine/crates/myfsio-server/src/services/access_logging.rs Normal file
View File

@@ -0,0 +1,105 @@
use parking_lot::RwLock;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LoggingConfiguration {
pub target_bucket: String,
#[serde(default)]
pub target_prefix: String,
#[serde(default = "default_enabled")]
pub enabled: bool,
}
fn default_enabled() -> bool {
true
}
#[derive(Serialize, Deserialize)]
struct StoredLoggingFile {
#[serde(rename = "LoggingEnabled")]
logging_enabled: Option<StoredLoggingEnabled>,
}
#[derive(Serialize, Deserialize)]
struct StoredLoggingEnabled {
#[serde(rename = "TargetBucket")]
target_bucket: String,
#[serde(rename = "TargetPrefix", default)]
target_prefix: String,
}
pub struct AccessLoggingService {
storage_root: PathBuf,
cache: RwLock<HashMap<String, Option<LoggingConfiguration>>>,
}
impl AccessLoggingService {
pub fn new(storage_root: &Path) -> Self {
Self {
storage_root: storage_root.to_path_buf(),
cache: RwLock::new(HashMap::new()),
}
}
fn config_path(&self, bucket: &str) -> PathBuf {
self.storage_root
.join(".myfsio.sys")
.join("buckets")
.join(bucket)
.join("logging.json")
}
pub fn get(&self, bucket: &str) -> Option<LoggingConfiguration> {
if let Some(cached) = self.cache.read().get(bucket).cloned() {
return cached;
}
let path = self.config_path(bucket);
let config = if path.exists() {
std::fs::read_to_string(&path)
.ok()
.and_then(|s| serde_json::from_str::<StoredLoggingFile>(&s).ok())
.and_then(|f| f.logging_enabled)
.map(|e| LoggingConfiguration {
target_bucket: e.target_bucket,
target_prefix: e.target_prefix,
enabled: true,
})
} else {
None
};
self.cache
.write()
.insert(bucket.to_string(), config.clone());
config
}
pub fn set(&self, bucket: &str, config: LoggingConfiguration) -> std::io::Result<()> {
let path = self.config_path(bucket);
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
let stored = StoredLoggingFile {
logging_enabled: Some(StoredLoggingEnabled {
target_bucket: config.target_bucket.clone(),
target_prefix: config.target_prefix.clone(),
}),
};
let json = serde_json::to_string_pretty(&stored)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
std::fs::write(&path, json)?;
self.cache.write().insert(bucket.to_string(), Some(config));
Ok(())
}
pub fn delete(&self, bucket: &str) {
let path = self.config_path(bucket);
if path.exists() {
let _ = std::fs::remove_file(&path);
}
self.cache.write().insert(bucket.to_string(), None);
}
}

35
rust/myfsio-engine/crates/myfsio-server/src/services/gc.rs
View File

@@ -28,6 +28,7 @@ pub struct GcService {
storage_root: PathBuf,
config: GcConfig,
running: Arc<RwLock<bool>>,
started_at: Arc<RwLock<Option<Instant>>>,
history: Arc<RwLock<Vec<Value>>>,
history_path: PathBuf,
}
@@ -53,6 +54,7 @@ impl GcService {
storage_root,
config,
running: Arc::new(RwLock::new(false)),
started_at: Arc::new(RwLock::new(None)),
history: Arc::new(RwLock::new(history)),
history_path,
}
@@ -60,9 +62,17 @@ impl GcService {
pub async fn status(&self) -> Value {
let running = *self.running.read().await;
let scan_elapsed_seconds = self
.started_at
.read()
.await
.as_ref()
.map(|started| started.elapsed().as_secs_f64());
json!({
"enabled": true,
"running": running,
"scanning": running,
"scan_elapsed_seconds": scan_elapsed_seconds,
"interval_hours": self.config.interval_hours,
"temp_file_max_age_hours": self.config.temp_file_max_age_hours,
"multipart_max_age_days": self.config.multipart_max_age_days,
@@ -73,7 +83,9 @@ impl GcService {
pub async fn history(&self) -> Value {
let history = self.history.read().await;
json!({ "executions": *history })
let mut executions: Vec<Value> = history.iter().cloned().collect();
executions.reverse();
json!({ "executions": executions })
}
pub async fn run_now(&self, dry_run: bool) -> Result<Value, String> {
@@ -84,12 +96,14 @@ impl GcService {
}
*running = true;
}
*self.started_at.write().await = Some(Instant::now());
let start = Instant::now();
let result = self.execute_gc(dry_run || self.config.dry_run).await;
let elapsed = start.elapsed().as_secs_f64();
*self.running.write().await = false;
*self.started_at.write().await = None;
let mut result_json = result.clone();
if let Some(obj) = result_json.as_object_mut() {
@@ -124,9 +138,12 @@ impl GcService {
let mut errors: Vec<String> = Vec::new();
let now = std::time::SystemTime::now();
let temp_max_age = std::time::Duration::from_secs_f64(self.config.temp_file_max_age_hours * 3600.0);
let multipart_max_age = std::time::Duration::from_secs(self.config.multipart_max_age_days * 86400);
let lock_max_age = std::time::Duration::from_secs_f64(self.config.lock_file_max_age_hours * 3600.0);
let temp_max_age =
std::time::Duration::from_secs_f64(self.config.temp_file_max_age_hours * 3600.0);
let multipart_max_age =
std::time::Duration::from_secs(self.config.multipart_max_age_days * 86400);
let lock_max_age =
std::time::Duration::from_secs_f64(self.config.lock_file_max_age_hours * 3600.0);
let tmp_dir = self.storage_root.join(".myfsio.sys").join("tmp");
if tmp_dir.exists() {
@@ -140,7 +157,10 @@ impl GcService {
let size = metadata.len();
if !dry_run {
if let Err(e) = std::fs::remove_file(entry.path()) {
errors.push(format!("Failed to remove temp file: {}", e));
errors.push(format!(
"Failed to remove temp file: {}",
e
));
continue;
}
}
@@ -242,7 +262,10 @@ impl GcService {
if let Some(parent) = self.history_path.parent() {
let _ = std::fs::create_dir_all(parent);
}
let _ = std::fs::write(&self.history_path, serde_json::to_string_pretty(&data).unwrap_or_default());
let _ = std::fs::write(
&self.history_path,
serde_json::to_string_pretty(&data).unwrap_or_default(),
);
}
pub fn start_background(self: Arc<Self>) -> tokio::task::JoinHandle<()> {

666
rust/myfsio-engine/crates/myfsio-server/src/services/integrity.rs
View File

@@ -1,11 +1,17 @@
use myfsio_common::constants::{
BUCKET_META_DIR, BUCKET_VERSIONS_DIR, INDEX_FILE, SYSTEM_BUCKETS_DIR, SYSTEM_ROOT,
};
use myfsio_storage::fs_backend::FsStorageBackend;
use myfsio_storage::traits::StorageEngine;
use serde_json::{json, Value};
use std::path::PathBuf;
use serde_json::{json, Map, Value};
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Instant;
use tokio::sync::RwLock;
const MAX_ISSUES: usize = 500;
const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"];
pub struct IntegrityConfig {
pub interval_hours: f64,
pub batch_size: usize,
@@ -17,7 +23,7 @@ impl Default for IntegrityConfig {
fn default() -> Self {
Self {
interval_hours: 24.0,
batch_size: 1000,
batch_size: 10_000,
auto_heal: false,
dry_run: false,
}
@@ -25,21 +31,70 @@ impl Default for IntegrityConfig {
}
pub struct IntegrityService {
#[allow(dead_code)]
storage: Arc<FsStorageBackend>,
storage_root: PathBuf,
config: IntegrityConfig,
running: Arc<RwLock<bool>>,
started_at: Arc<RwLock<Option<Instant>>>,
history: Arc<RwLock<Vec<Value>>>,
history_path: PathBuf,
}
#[derive(Default)]
struct ScanState {
objects_scanned: u64,
buckets_scanned: u64,
corrupted_objects: u64,
orphaned_objects: u64,
phantom_metadata: u64,
stale_versions: u64,
etag_cache_inconsistencies: u64,
issues: Vec<Value>,
errors: Vec<String>,
}
impl ScanState {
fn batch_exhausted(&self, batch_size: usize) -> bool {
self.objects_scanned >= batch_size as u64
}
fn push_issue(&mut self, issue_type: &str, bucket: &str, key: &str, detail: String) {
if self.issues.len() < MAX_ISSUES {
self.issues.push(json!({
"issue_type": issue_type,
"bucket": bucket,
"key": key,
"detail": detail,
}));
}
}
fn into_json(self, elapsed: f64) -> Value {
json!({
"objects_scanned": self.objects_scanned,
"buckets_scanned": self.buckets_scanned,
"corrupted_objects": self.corrupted_objects,
"orphaned_objects": self.orphaned_objects,
"phantom_metadata": self.phantom_metadata,
"stale_versions": self.stale_versions,
"etag_cache_inconsistencies": self.etag_cache_inconsistencies,
"issues_healed": 0,
"issues": self.issues,
"errors": self.errors,
"execution_time_seconds": elapsed,
})
}
}
impl IntegrityService {
pub fn new(
storage: Arc<FsStorageBackend>,
storage_root: &std::path::Path,
storage_root: &Path,
config: IntegrityConfig,
) -> Self {
let history_path = storage_root
.join(".myfsio.sys")
.join(SYSTEM_ROOT)
.join("config")
.join("integrity_history.json");
@@ -55,8 +110,10 @@ impl IntegrityService {
Self {
storage,
storage_root: storage_root.to_path_buf(),
config,
running: Arc::new(RwLock::new(false)),
started_at: Arc::new(RwLock::new(None)),
history: Arc::new(RwLock::new(history)),
history_path,
}
@@ -64,9 +121,17 @@ impl IntegrityService {
pub async fn status(&self) -> Value {
let running = *self.running.read().await;
let scan_elapsed_seconds = self
.started_at
.read()
.await
.as_ref()
.map(|started| started.elapsed().as_secs_f64());
json!({
"enabled": true,
"running": running,
"scanning": running,
"scan_elapsed_seconds": scan_elapsed_seconds,
"interval_hours": self.config.interval_hours,
"batch_size": self.config.batch_size,
"auto_heal": self.config.auto_heal,
@@ -76,7 +141,9 @@ impl IntegrityService {
pub async fn history(&self) -> Value {
let history = self.history.read().await;
json!({ "executions": *history })
let mut executions: Vec<Value> = history.iter().cloned().collect();
executions.reverse();
json!({ "executions": executions })
}
pub async fn run_now(&self, dry_run: bool, auto_heal: bool) -> Result<Value, String> {
@@ -87,23 +154,31 @@ impl IntegrityService {
}
*running = true;
}
*self.started_at.write().await = Some(Instant::now());
let start = Instant::now();
let result = self.check_integrity(dry_run, auto_heal).await;
let storage_root = self.storage_root.clone();
let batch_size = self.config.batch_size;
let result =
tokio::task::spawn_blocking(move || scan_all_buckets(&storage_root, batch_size))
.await
.unwrap_or_else(|e| {
let mut st = ScanState::default();
st.errors.push(format!("scan task failed: {}", e));
st
});
let elapsed = start.elapsed().as_secs_f64();
*self.running.write().await = false;
*self.started_at.write().await = None;
let mut result_json = result.clone();
if let Some(obj) = result_json.as_object_mut() {
obj.insert("execution_time_seconds".to_string(), json!(elapsed));
}
let result_json = result.into_json(elapsed);
let record = json!({
"timestamp": chrono::Utc::now().timestamp_millis() as f64 / 1000.0,
"dry_run": dry_run,
"auto_heal": auto_heal,
"result": result_json,
"result": result_json.clone(),
});
{
@@ -116,62 +191,7 @@ impl IntegrityService {
}
self.save_history().await;
Ok(result)
}
async fn check_integrity(&self, _dry_run: bool, _auto_heal: bool) -> Value {
let buckets = match self.storage.list_buckets().await {
Ok(b) => b,
Err(e) => return json!({"error": e.to_string()}),
};
let mut objects_scanned = 0u64;
let mut corrupted = 0u64;
let mut phantom_metadata = 0u64;
let mut errors: Vec<String> = Vec::new();
for bucket in &buckets {
let params = myfsio_common::types::ListParams {
max_keys: self.config.batch_size,
..Default::default()
};
let objects = match self.storage.list_objects(&bucket.name, &params).await {
Ok(r) => r.objects,
Err(e) => {
errors.push(format!("{}: {}", bucket.name, e));
continue;
}
};
for obj in &objects {
objects_scanned += 1;
match self.storage.get_object_path(&bucket.name, &obj.key).await {
Ok(path) => {
if !path.exists() {
phantom_metadata += 1;
} else if let Some(ref expected_etag) = obj.etag {
match myfsio_crypto::hashing::md5_file(&path) {
Ok(actual_etag) => {
if &actual_etag != expected_etag {
corrupted += 1;
}
}
Err(e) => errors.push(format!("{}:{}: {}", bucket.name, obj.key, e)),
}
}
}
Err(e) => errors.push(format!("{}:{}: {}", bucket.name, obj.key, e)),
}
}
}
json!({
"objects_scanned": objects_scanned,
"buckets_scanned": buckets.len(),
"corrupted_objects": corrupted,
"phantom_metadata": phantom_metadata,
"errors": errors,
})
Ok(result_json)
}
async fn save_history(&self) {
@@ -202,3 +222,511 @@ impl IntegrityService {
})
}
}
fn scan_all_buckets(storage_root: &Path, batch_size: usize) -> ScanState {
let mut state = ScanState::default();
let buckets = match list_bucket_names(storage_root) {
Ok(b) => b,
Err(e) => {
state.errors.push(format!("list buckets: {}", e));
return state;
}
};
for bucket in &buckets {
if state.batch_exhausted(batch_size) {
break;
}
state.buckets_scanned += 1;
let bucket_path = storage_root.join(bucket);
let meta_root = storage_root
.join(SYSTEM_ROOT)
.join(SYSTEM_BUCKETS_DIR)
.join(bucket)
.join(BUCKET_META_DIR);
let index_entries = collect_index_entries(&meta_root);
check_corrupted(&mut state, bucket, &bucket_path, &index_entries, batch_size);
check_phantom(&mut state, bucket, &bucket_path, &index_entries, batch_size);
check_orphaned(&mut state, bucket, &bucket_path, &index_entries, batch_size);
check_stale_versions(&mut state, storage_root, bucket, batch_size);
check_etag_cache(&mut state, storage_root, bucket, &index_entries, batch_size);
}
state
}
fn list_bucket_names(storage_root: &Path) -> std::io::Result<Vec<String>> {
let mut names = Vec::new();
if !storage_root.exists() {
return Ok(names);
}
for entry in std::fs::read_dir(storage_root)? {
let entry = entry?;
let name = entry.file_name().to_string_lossy().to_string();
if name == SYSTEM_ROOT {
continue;
}
if entry.file_type().map(|t| t.is_dir()).unwrap_or(false) {
names.push(name);
}
}
Ok(names)
}
#[allow(dead_code)]
struct IndexEntryInfo {
entry: Value,
index_file: PathBuf,
key_name: String,
}
fn collect_index_entries(meta_root: &Path) -> HashMap<String, IndexEntryInfo> {
let mut out: HashMap<String, IndexEntryInfo> = HashMap::new();
if !meta_root.exists() {
return out;
}
let mut stack: Vec<PathBuf> = vec![meta_root.to_path_buf()];
while let Some(dir) = stack.pop() {
let rd = match std::fs::read_dir(&dir) {
Ok(r) => r,
Err(_) => continue,
};
for entry in rd.flatten() {
let path = entry.path();
let ft = match entry.file_type() {
Ok(t) => t,
Err(_) => continue,
};
if ft.is_dir() {
stack.push(path);
continue;
}
if entry.file_name().to_string_lossy() != INDEX_FILE {
continue;
}
let rel_dir = match path.parent().and_then(|p| p.strip_prefix(meta_root).ok()) {
Some(p) => p.to_path_buf(),
None => continue,
};
let dir_prefix = if rel_dir.as_os_str().is_empty() {
String::new()
} else {
rel_dir
.components()
.map(|c| c.as_os_str().to_string_lossy().to_string())
.collect::<Vec<_>>()
.join("/")
};
let content = match std::fs::read_to_string(&path) {
Ok(c) => c,
Err(_) => continue,
};
let index_data: Map<String, Value> = match serde_json::from_str(&content) {
Ok(Value::Object(m)) => m,
_ => continue,
};
for (key_name, entry_val) in index_data {
let full_key = if dir_prefix.is_empty() {
key_name.clone()
} else {
format!("{}/{}", dir_prefix, key_name)
};
out.insert(
full_key,
IndexEntryInfo {
entry: entry_val,
index_file: path.clone(),
key_name,
},
);
}
}
}
out
}
fn stored_etag(entry: &Value) -> Option<String> {
entry
.get("metadata")
.and_then(|m| m.get("__etag__"))
.and_then(|v| v.as_str())
.map(|s| s.to_string())
}
fn check_corrupted(
state: &mut ScanState,
bucket: &str,
bucket_path: &Path,
entries: &HashMap<String, IndexEntryInfo>,
batch_size: usize,
) {
let mut keys: Vec<&String> = entries.keys().collect();
keys.sort();
for full_key in keys {
if state.batch_exhausted(batch_size) {
return;
}
let info = &entries[full_key];
let object_path = bucket_path.join(full_key);
if !object_path.exists() {
continue;
}
state.objects_scanned += 1;
let Some(stored) = stored_etag(&info.entry) else {
continue;
};
match myfsio_crypto::hashing::md5_file(&object_path) {
Ok(actual) => {
if actual != stored {
state.corrupted_objects += 1;
state.push_issue(
"corrupted_object",
bucket,
full_key,
format!("stored_etag={} actual_etag={}", stored, actual),
);
}
}
Err(e) => state
.errors
.push(format!("hash {}/{}: {}", bucket, full_key, e)),
}
}
}
fn check_phantom(
state: &mut ScanState,
bucket: &str,
bucket_path: &Path,
entries: &HashMap<String, IndexEntryInfo>,
batch_size: usize,
) {
let mut keys: Vec<&String> = entries.keys().collect();
keys.sort();
for full_key in keys {
if state.batch_exhausted(batch_size) {
return;
}
state.objects_scanned += 1;
let object_path = bucket_path.join(full_key);
if !object_path.exists() {
state.phantom_metadata += 1;
state.push_issue(
"phantom_metadata",
bucket,
full_key,
"metadata entry without file on disk".to_string(),
);
}
}
}
fn check_orphaned(
state: &mut ScanState,
bucket: &str,
bucket_path: &Path,
entries: &HashMap<String, IndexEntryInfo>,
batch_size: usize,
) {
let indexed: HashSet<&String> = entries.keys().collect();
let mut stack: Vec<(PathBuf, String)> = vec![(bucket_path.to_path_buf(), String::new())];
while let Some((dir, prefix)) = stack.pop() {
if state.batch_exhausted(batch_size) {
return;
}
let rd = match std::fs::read_dir(&dir) {
Ok(r) => r,
Err(_) => continue,
};
for entry in rd.flatten() {
if state.batch_exhausted(batch_size) {
return;
}
let name = entry.file_name().to_string_lossy().to_string();
let ft = match entry.file_type() {
Ok(t) => t,
Err(_) => continue,
};
if ft.is_dir() {
if prefix.is_empty() && INTERNAL_FOLDERS.contains(&name.as_str()) {
continue;
}
let new_prefix = if prefix.is_empty() {
name
} else {
format!("{}/{}", prefix, name)
};
stack.push((entry.path(), new_prefix));
} else if ft.is_file() {
let full_key = if prefix.is_empty() {
name
} else {
format!("{}/{}", prefix, name)
};
state.objects_scanned += 1;
if !indexed.contains(&full_key) {
state.orphaned_objects += 1;
state.push_issue(
"orphaned_object",
bucket,
&full_key,
"file exists without metadata entry".to_string(),
);
}
}
}
}
}
fn check_stale_versions(
state: &mut ScanState,
storage_root: &Path,
bucket: &str,
batch_size: usize,
) {
let versions_root = storage_root
.join(SYSTEM_ROOT)
.join(SYSTEM_BUCKETS_DIR)
.join(bucket)
.join(BUCKET_VERSIONS_DIR);
if !versions_root.exists() {
return;
}
let mut stack: Vec<PathBuf> = vec![versions_root.clone()];
while let Some(dir) = stack.pop() {
if state.batch_exhausted(batch_size) {
return;
}
let rd = match std::fs::read_dir(&dir) {
Ok(r) => r,
Err(_) => continue,
};
let mut bin_stems: HashMap<String, PathBuf> = HashMap::new();
let mut json_stems: HashMap<String, PathBuf> = HashMap::new();
let mut subdirs: Vec<PathBuf> = Vec::new();
for entry in rd.flatten() {
let ft = match entry.file_type() {
Ok(t) => t,
Err(_) => continue,
};
let path = entry.path();
if ft.is_dir() {
subdirs.push(path);
continue;
}
let name = entry.file_name().to_string_lossy().to_string();
if let Some(stem) = name.strip_suffix(".bin") {
bin_stems.insert(stem.to_string(), path);
} else if let Some(stem) = name.strip_suffix(".json") {
json_stems.insert(stem.to_string(), path);
}
}
for (stem, path) in &bin_stems {
if state.batch_exhausted(batch_size) {
return;
}
state.objects_scanned += 1;
if !json_stems.contains_key(stem) {
state.stale_versions += 1;
let key = path
.strip_prefix(&versions_root)
.map(|p| p.to_string_lossy().replace('\\', "/"))
.unwrap_or_else(|_| path.display().to_string());
state.push_issue(
"stale_version",
bucket,
&key,
"version data without manifest".to_string(),
);
}
}
for (stem, path) in &json_stems {
if state.batch_exhausted(batch_size) {
return;
}
state.objects_scanned += 1;
if !bin_stems.contains_key(stem) {
state.stale_versions += 1;
let key = path
.strip_prefix(&versions_root)
.map(|p| p.to_string_lossy().replace('\\', "/"))
.unwrap_or_else(|_| path.display().to_string());
state.push_issue(
"stale_version",
bucket,
&key,
"version manifest without data".to_string(),
);
}
}
stack.extend(subdirs);
}
}
fn check_etag_cache(
state: &mut ScanState,
storage_root: &Path,
bucket: &str,
entries: &HashMap<String, IndexEntryInfo>,
batch_size: usize,
) {
let etag_index_path = storage_root
.join(SYSTEM_ROOT)
.join(SYSTEM_BUCKETS_DIR)
.join(bucket)
.join("etag_index.json");
if !etag_index_path.exists() {
return;
}
let cache: HashMap<String, Value> = match std::fs::read_to_string(&etag_index_path)
.ok()
.and_then(|s| serde_json::from_str(&s).ok())
{
Some(Value::Object(m)) => m.into_iter().collect(),
_ => return,
};
for (full_key, cached_val) in cache {
if state.batch_exhausted(batch_size) {
return;
}
state.objects_scanned += 1;
let Some(cached_etag) = cached_val.as_str() else {
continue;
};
let Some(info) = entries.get(&full_key) else {
continue;
};
let Some(stored) = stored_etag(&info.entry) else {
continue;
};
if cached_etag != stored {
state.etag_cache_inconsistencies += 1;
state.push_issue(
"etag_cache_inconsistency",
bucket,
&full_key,
format!("cached_etag={} index_etag={}", cached_etag, stored),
);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
fn md5_hex(bytes: &[u8]) -> String {
myfsio_crypto::hashing::md5_bytes(bytes)
}
fn write_index(meta_dir: &Path, entries: &[(&str, &str)]) {
fs::create_dir_all(meta_dir).unwrap();
let mut map = Map::new();
for (name, etag) in entries {
map.insert(
name.to_string(),
json!({ "metadata": { "__etag__": etag } }),
);
}
fs::write(
meta_dir.join(INDEX_FILE),
serde_json::to_string(&Value::Object(map)).unwrap(),
)
.unwrap();
}
#[test]
fn scan_detects_each_issue_type() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
let bucket = "testbucket";
let bucket_path = root.join(bucket);
let meta_root = root
.join(SYSTEM_ROOT)
.join(SYSTEM_BUCKETS_DIR)
.join(bucket)
.join(BUCKET_META_DIR);
fs::create_dir_all(&bucket_path).unwrap();
let clean_bytes = b"clean file contents";
let clean_etag = md5_hex(clean_bytes);
fs::write(bucket_path.join("clean.txt"), clean_bytes).unwrap();
let corrupted_bytes = b"actual content";
fs::write(bucket_path.join("corrupted.txt"), corrupted_bytes).unwrap();
fs::write(bucket_path.join("orphan.txt"), b"no metadata").unwrap();
write_index(
&meta_root,
&[
("clean.txt", &clean_etag),
("corrupted.txt", "00000000000000000000000000000000"),
("phantom.txt", "deadbeefdeadbeefdeadbeefdeadbeef"),
],
);
let versions_root = root
.join(SYSTEM_ROOT)
.join(SYSTEM_BUCKETS_DIR)
.join(bucket)
.join(BUCKET_VERSIONS_DIR)
.join("someobject");
fs::create_dir_all(&versions_root).unwrap();
fs::write(versions_root.join("v1.bin"), b"orphan bin").unwrap();
fs::write(versions_root.join("v2.json"), b"{}").unwrap();
let etag_index = root
.join(SYSTEM_ROOT)
.join(SYSTEM_BUCKETS_DIR)
.join(bucket)
.join("etag_index.json");
fs::write(
&etag_index,
serde_json::to_string(&json!({ "clean.txt": "stale-cached-etag" })).unwrap(),
)
.unwrap();
let state = scan_all_buckets(root, 10_000);
assert_eq!(state.corrupted_objects, 1, "corrupted");
assert_eq!(state.phantom_metadata, 1, "phantom");
assert_eq!(state.orphaned_objects, 1, "orphaned");
assert_eq!(state.stale_versions, 2, "stale versions");
assert_eq!(state.etag_cache_inconsistencies, 1, "etag cache");
assert_eq!(state.buckets_scanned, 1);
assert!(
state.errors.is_empty(),
"unexpected errors: {:?}",
state.errors
);
}
#[test]
fn skips_system_root_as_bucket() {
let tmp = tempfile::tempdir().unwrap();
fs::create_dir_all(tmp.path().join(SYSTEM_ROOT).join("config")).unwrap();
let state = scan_all_buckets(tmp.path(), 100);
assert_eq!(state.buckets_scanned, 0);
}
}

26
rust/myfsio-engine/crates/myfsio-server/src/services/lifecycle.rs
View File

@@ -66,7 +66,10 @@ impl LifecycleService {
None => continue,
};
let rules = match lifecycle.as_str().and_then(|s| serde_json::from_str::<Value>(s).ok()) {
let rules = match lifecycle
.as_str()
.and_then(|s| serde_json::from_str::<Value>(s).ok())
{
Some(v) => v,
None => continue,
};
@@ -93,7 +96,11 @@ impl LifecycleService {
let cutoff = chrono::Utc::now() - chrono::Duration::days(days as i64);
let params = myfsio_common::types::ListParams {
max_keys: 1000,
prefix: if prefix.is_empty() { None } else { Some(prefix.to_string()) },
prefix: if prefix.is_empty() {
None
} else {
Some(prefix.to_string())
},
..Default::default()
};
if let Ok(result) = self.storage.list_objects(&bucket.name, &params).await {
@@ -101,7 +108,8 @@ impl LifecycleService {
if obj.last_modified < cutoff {
match self.storage.delete_object(&bucket.name, &obj.key).await {
Ok(()) => total_expired += 1,
Err(e) => errors.push(format!("{}:{}: {}", bucket.name, obj.key, e)),
Err(e) => errors
.push(format!("{}:{}: {}", bucket.name, obj.key, e)),
}
}
}
@@ -112,12 +120,18 @@ impl LifecycleService {
if let Some(abort) = rule.get("AbortIncompleteMultipartUpload") {
if let Some(days) = abort.get("DaysAfterInitiation").and_then(|d| d.as_u64()) {
let cutoff = chrono::Utc::now() - chrono::Duration::days(days as i64);
if let Ok(uploads) = self.storage.list_multipart_uploads(&bucket.name).await {
if let Ok(uploads) = self.storage.list_multipart_uploads(&bucket.name).await
{
for upload in &uploads {
if upload.initiated < cutoff {
match self.storage.abort_multipart(&bucket.name, &upload.upload_id).await {
match self
.storage
.abort_multipart(&bucket.name, &upload.upload_id)
.await
{
Ok(()) => total_multipart_aborted += 1,
Err(e) => errors.push(format!("abort {}: {}", upload.upload_id, e)),
Err(e) => errors
.push(format!("abort {}: {}", upload.upload_id, e)),
}
}
}

9
rust/myfsio-engine/crates/myfsio-server/src/services/metrics.rs
View File

@@ -165,8 +165,9 @@ impl MetricsService {
.ok()
.and_then(|s| serde_json::from_str::<Value>(&s).ok())
.and_then(|v| {
v.get("snapshots")
.and_then(|s| serde_json::from_value::<Vec<MetricsSnapshot>>(s.clone()).ok())
v.get("snapshots").and_then(|s| {
serde_json::from_value::<Vec<MetricsSnapshot>>(s.clone()).ok()
})
})
.unwrap_or_default()
} else {
@@ -218,7 +219,9 @@ impl MetricsService {
if let Some(code) = error_code {
*inner.error_codes.entry(code.to_string()).or_insert(0) += 1;
}
inner.totals.record(latency_ms, success, bytes_in, bytes_out);
inner
.totals
.record(latency_ms, success, bytes_in, bytes_out);
}
pub fn get_current_stats(&self) -> Value {

4
rust/myfsio-engine/crates/myfsio-server/src/services/mod.rs
View File

@@ -1,9 +1,11 @@
pub mod access_logging;
pub mod gc;
pub mod lifecycle;
pub mod integrity;
pub mod lifecycle;
pub mod metrics;
pub mod replication;
pub mod s3_client;
pub mod site_registry;
pub mod site_sync;
pub mod system_metrics;
pub mod website_domains;

145
rust/myfsio-engine/crates/myfsio-server/src/services/replication.rs
View File

@@ -8,6 +8,7 @@ use parking_lot::Mutex;
use serde::{Deserialize, Serialize};
use tokio::sync::Semaphore;
use myfsio_common::types::ListParams;
use myfsio_storage::fs_backend::FsStorageBackend;
use myfsio_storage::traits::StorageEngine;
@@ -124,7 +125,10 @@ impl ReplicationFailureStore {
}
let trimmed = &failures[..failures.len().min(self.max_failures_per_bucket)];
let data = serde_json::json!({ "failures": trimmed });
let _ = std::fs::write(&path, serde_json::to_string_pretty(&data).unwrap_or_default());
let _ = std::fs::write(
&path,
serde_json::to_string_pretty(&data).unwrap_or_default(),
);
}
pub fn load(&self, bucket: &str) -> Vec<ReplicationFailure> {
@@ -148,7 +152,10 @@ impl ReplicationFailureStore {
pub fn add(&self, bucket: &str, failure: ReplicationFailure) {
let mut failures = self.load(bucket);
if let Some(existing) = failures.iter_mut().find(|f| f.object_key == failure.object_key) {
if let Some(existing) = failures
.iter_mut()
.find(|f| f.object_key == failure.object_key)
{
existing.failure_count += 1;
existing.timestamp = failure.timestamp;
existing.error_message = failure.error_message.clone();
@@ -318,7 +325,101 @@ impl ReplicationManager {
let manager = self.clone();
tokio::spawn(async move {
let _permit = permit;
manager.replicate_task(&bucket, &key, &rule, &connection, &action).await;
manager
.replicate_task(&bucket, &key, &rule, &connection, &action)
.await;
});
}
pub async fn replicate_existing_objects(self: Arc<Self>, bucket: String) -> usize {
let rule = match self.get_rule(&bucket) {
Some(r) if r.enabled => r,
_ => return 0,
};
let connection = match self.connections.get(&rule.target_connection_id) {
Some(c) => c,
None => {
tracing::warn!(
"Cannot replicate existing objects for {}: connection {} not found",
bucket,
rule.target_connection_id
);
return 0;
}
};
if !self.check_endpoint(&connection).await {
tracing::warn!(
"Cannot replicate existing objects for {}: endpoint {} is unreachable",
bucket,
connection.endpoint_url
);
return 0;
}
let mut continuation_token: Option<String> = None;
let mut submitted = 0usize;
loop {
let page = match self
.storage
.list_objects(
&bucket,
&ListParams {
max_keys: 1000,
continuation_token: continuation_token.clone(),
prefix: rule.filter_prefix.clone(),
start_after: None,
},
)
.await
{
Ok(page) => page,
Err(err) => {
tracing::error!(
"Failed to list existing objects for replication in {}: {}",
bucket,
err
);
break;
}
};
let next_token = page.next_continuation_token.clone();
let is_truncated = page.is_truncated;
for object in page.objects {
submitted += 1;
self.clone()
.trigger(bucket.clone(), object.key, "write".to_string())
.await;
}
if !is_truncated {
break;
}
continuation_token = next_token;
if continuation_token.is_none() {
break;
}
}
submitted
}
pub fn schedule_existing_objects_sync(self: Arc<Self>, bucket: String) {
tokio::spawn(async move {
let submitted = self
.clone()
.replicate_existing_objects(bucket.clone())
.await;
if submitted > 0 {
tracing::info!(
"Scheduled {} existing object(s) for replication in {}",
submitted,
bucket
);
}
});
}
@@ -330,7 +431,8 @@ impl ReplicationManager {
conn: &RemoteConnection,
action: &str,
) {
if object_key.contains("..") || object_key.starts_with('/') || object_key.starts_with('\\') {
if object_key.contains("..") || object_key.starts_with('/') || object_key.starts_with('\\')
{
tracing::error!("Invalid object key (path traversal): {}", object_key);
return;
}
@@ -358,7 +460,12 @@ impl ReplicationManager {
}
Err(err) => {
let msg = format!("{:?}", err);
tracing::error!("Replication DELETE failed {}/{}: {}", bucket, object_key, msg);
tracing::error!(
"Replication DELETE failed {}/{}: {}",
bucket,
object_key,
msg
);
self.failures.add(
bucket,
ReplicationFailure {
@@ -414,16 +521,18 @@ impl ReplicationManager {
.send()
.await
{
Ok(_) | Err(_) => upload_object(
&client,
&rule.target_bucket,
object_key,
&src_path,
file_size,
self.streaming_threshold_bytes,
content_type.as_deref(),
)
.await,
Ok(_) | Err(_) => {
upload_object(
&client,
&rule.target_bucket,
object_key,
&src_path,
file_size,
self.streaming_threshold_bytes,
content_type.as_deref(),
)
.await
}
}
}
other => other,
@@ -577,9 +686,9 @@ async fn upload_object(
)))
})?
} else {
let bytes = tokio::fs::read(path).await.map_err(|e| {
aws_sdk_s3::error::SdkError::construction_failure(Box::new(e))
})?;
let bytes = tokio::fs::read(path)
.await
.map_err(|e| aws_sdk_s3::error::SdkError::construction_failure(Box::new(e)))?;
ByteStream::from(bytes)
};

4
rust/myfsio-engine/crates/myfsio-server/src/services/s3_client.rs
View File

@@ -37,8 +37,8 @@ pub fn build_client(connection: &RemoteConnection, options: &ClientOptions) -> C
.read_timeout(options.read_timeout)
.build();
let retry_config = aws_smithy_types::retry::RetryConfig::standard()
.with_max_attempts(options.max_attempts);
let retry_config =
aws_smithy_types::retry::RetryConfig::standard().with_max_attempts(options.max_attempts);
let config = aws_sdk_s3::config::Builder::new()
.behavior_version(BehaviorVersion::latest())

7
rust/myfsio-engine/crates/myfsio-server/src/services/site_registry.rs
View File

@@ -102,7 +102,12 @@ impl SiteRegistry {
}
pub fn get_peer(&self, site_id: &str) -> Option<PeerSite> {
self.data.read().peers.iter().find(|p| p.site_id == site_id).cloned()
self.data
.read()
.peers
.iter()
.find(|p| p.site_id == site_id)
.cloned()
}
pub fn add_peer(&self, peer: PeerSite) {

33
rust/myfsio-engine/crates/myfsio-server/src/services/site_sync.rs
View File

@@ -102,7 +102,10 @@ impl SiteSyncWorker {
}
pub async fn run(self: Arc<Self>) {
tracing::info!("Site sync worker started (interval={}s)", self.interval.as_secs());
tracing::info!(
"Site sync worker started (interval={}s)",
self.interval.as_secs()
);
loop {
tokio::select! {
_ = tokio::time::sleep(self.interval) => {}
@@ -309,11 +312,10 @@ impl SiteSyncWorker {
let resp = match req.send().await {
Ok(r) => r,
Err(err) => {
let msg = format!("{:?}", err);
if msg.contains("NoSuchBucket") {
if is_not_found_error(&err) {
return Ok(result);
}
return Err(msg);
return Err(format!("{:?}", err));
}
};
for obj in resp.contents() {
@@ -409,11 +411,9 @@ impl SiteSyncWorker {
}
};
let metadata: Option<HashMap<String, String>> = head.metadata().map(|m| {
m.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect()
});
let metadata: Option<HashMap<String, String>> = head
.metadata()
.map(|m| m.iter().map(|(k, v)| (k.clone(), v.clone())).collect());
let stream = resp.body.into_async_read();
let boxed: Pin<Box<dyn AsyncRead + Send>> = Box::pin(stream);
@@ -428,7 +428,12 @@ impl SiteSyncWorker {
true
}
Err(err) => {
tracing::error!("Store pulled object failed {}/{}: {}", local_bucket, key, err);
tracing::error!(
"Store pulled object failed {}/{}: {}",
local_bucket,
key,
err
);
false
}
}
@@ -483,3 +488,11 @@ fn now_secs() -> f64 {
.map(|d| d.as_secs_f64())
.unwrap_or(0.0)
}
fn is_not_found_error<E: std::fmt::Debug>(err: &aws_sdk_s3::error::SdkError<E>) -> bool {
let msg = format!("{:?}", err);
msg.contains("NoSuchBucket")
|| msg.contains("code: Some(\"NotFound\")")
|| msg.contains("code: Some(\"NoSuchBucket\")")
|| msg.contains("status: 404")
}

203
rust/myfsio-engine/crates/myfsio-server/src/services/system_metrics.rs Normal file
View File

@@ -0,0 +1,203 @@
use chrono::{DateTime, Utc};
use myfsio_storage::fs_backend::FsStorageBackend;
use myfsio_storage::traits::StorageEngine;
use serde::{Deserialize, Serialize};
use serde_json::json;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use sysinfo::{Disks, System};
use tokio::sync::RwLock;
#[derive(Debug, Clone)]
pub struct SystemMetricsConfig {
pub interval_minutes: u64,
pub retention_hours: u64,
}
impl Default for SystemMetricsConfig {
fn default() -> Self {
Self {
interval_minutes: 5,
retention_hours: 24,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SystemMetricsSnapshot {
pub timestamp: DateTime<Utc>,
pub cpu_percent: f64,
pub memory_percent: f64,
pub disk_percent: f64,
pub storage_bytes: u64,
}
pub struct SystemMetricsService {
storage_root: PathBuf,
storage: Arc<FsStorageBackend>,
config: SystemMetricsConfig,
history: Arc<RwLock<Vec<SystemMetricsSnapshot>>>,
history_path: PathBuf,
}
impl SystemMetricsService {
pub fn new(
storage_root: &Path,
storage: Arc<FsStorageBackend>,
config: SystemMetricsConfig,
) -> Self {
let history_path = storage_root
.join(".myfsio.sys")
.join("config")
.join("metrics_history.json");
let mut history = if history_path.exists() {
std::fs::read_to_string(&history_path)
.ok()
.and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
.and_then(|v| {
v.get("history").and_then(|h| {
serde_json::from_value::<Vec<SystemMetricsSnapshot>>(h.clone()).ok()
})
})
.unwrap_or_default()
} else {
Vec::new()
};
prune_history(&mut history, config.retention_hours);
Self {
storage_root: storage_root.to_path_buf(),
storage,
config,
history: Arc::new(RwLock::new(history)),
history_path,
}
}
pub async fn get_history(&self, hours: Option<u64>) -> Vec<SystemMetricsSnapshot> {
let mut history = self.history.read().await.clone();
prune_history(&mut history, hours.unwrap_or(self.config.retention_hours));
history
}
async fn take_snapshot(&self) {
let snapshot = collect_snapshot(&self.storage_root, &self.storage).await;
let mut history = self.history.write().await;
history.push(snapshot);
prune_history(&mut history, self.config.retention_hours);
drop(history);
self.save_history().await;
}
async fn save_history(&self) {
let history = self.history.read().await;
let data = json!({ "history": *history });
if let Some(parent) = self.history_path.parent() {
let _ = std::fs::create_dir_all(parent);
}
let _ = std::fs::write(
&self.history_path,
serde_json::to_string_pretty(&data).unwrap_or_default(),
);
}
pub fn start_background(self: Arc<Self>) -> tokio::task::JoinHandle<()> {
let interval =
std::time::Duration::from_secs(self.config.interval_minutes.saturating_mul(60));
tokio::spawn(async move {
self.take_snapshot().await;
let mut timer = tokio::time::interval(interval);
loop {
timer.tick().await;
self.take_snapshot().await;
}
})
}
}
fn prune_history(history: &mut Vec<SystemMetricsSnapshot>, retention_hours: u64) {
let cutoff = Utc::now() - chrono::Duration::hours(retention_hours as i64);
history.retain(|item| item.timestamp > cutoff);
}
fn sample_system_now() -> (f64, f64) {
let mut system = System::new();
system.refresh_cpu_usage();
std::thread::sleep(sysinfo::MINIMUM_CPU_UPDATE_INTERVAL);
system.refresh_cpu_usage();
system.refresh_memory();
let cpu_percent = system.global_cpu_usage() as f64;
let memory_percent = if system.total_memory() > 0 {
(system.used_memory() as f64 / system.total_memory() as f64) * 100.0
} else {
0.0
};
(cpu_percent, memory_percent)
}
fn normalize_path_for_mount(path: &Path) -> String {
let canonical = path.canonicalize().unwrap_or_else(|_| path.to_path_buf());
let raw = canonical.to_string_lossy().to_string();
let stripped = raw.strip_prefix(r"\\?\").unwrap_or(&raw);
stripped.to_lowercase()
}
fn sample_disk(path: &Path) -> (u64, u64) {
let disks = Disks::new_with_refreshed_list();
let path_str = normalize_path_for_mount(path);
let mut best: Option<(usize, u64, u64)> = None;
for disk in disks.list() {
let mount_raw = disk.mount_point().to_string_lossy().to_string();
let mount = mount_raw
.strip_prefix(r"\\?\")
.unwrap_or(&mount_raw)
.to_lowercase();
let total = disk.total_space();
let free = disk.available_space();
if path_str.starts_with(&mount) {
let len = mount.len();
match best {
Some((best_len, _, _)) if len <= best_len => {}
_ => best = Some((len, total, free)),
}
}
}
best.map(|(_, total, free)| (total, free)).unwrap_or((0, 0))
}
async fn collect_snapshot(
storage_root: &Path,
storage: &Arc<FsStorageBackend>,
) -> SystemMetricsSnapshot {
let (cpu_percent, memory_percent) = sample_system_now();
let (disk_total, disk_free) = sample_disk(storage_root);
let disk_percent = if disk_total > 0 {
((disk_total - disk_free) as f64 / disk_total as f64) * 100.0
} else {
0.0
};
let mut storage_bytes = 0u64;
let buckets = storage.list_buckets().await.unwrap_or_default();
for bucket in buckets {
if let Ok(stats) = storage.bucket_stats(&bucket.name).await {
storage_bytes += stats.total_bytes();
}
}
SystemMetricsSnapshot {
timestamp: Utc::now(),
cpu_percent: round2(cpu_percent),
memory_percent: round2(memory_percent),
disk_percent: round2(disk_percent),
storage_bytes,
}
}
fn round2(value: f64) -> f64 {
(value * 100.0).round() / 100.0
}

5
rust/myfsio-engine/crates/myfsio-server/src/services/website_domains.rs
View File

@@ -64,7 +64,10 @@ impl WebsiteDomainStore {
}
pub fn set_mapping(&self, domain: &str, bucket: &str) {
self.data.write().mappings.insert(domain.to_string(), bucket.to_string());
self.data
.write()
.mappings
.insert(domain.to_string(), bucket.to_string());
self.save();
}

5
rust/myfsio-engine/crates/myfsio-server/src/session.rs
View File

@@ -27,21 +27,18 @@ pub struct SessionData {
pub csrf_token: String,
pub flash: Vec<FlashMessage>,
pub extra: HashMap<String, String>,
created_at: Instant,
last_accessed: Instant,
}
impl SessionData {
pub fn new() -> Self {
let now = Instant::now();
Self {
user_id: None,
display_name: None,
csrf_token: generate_token(CSRF_TOKEN_BYTES),
flash: Vec::new(),
extra: HashMap::new(),
created_at: now,
last_accessed: now,
last_accessed: Instant::now(),
}
}

33
rust/myfsio-engine/crates/myfsio-server/src/state.rs
View File

@@ -2,16 +2,18 @@ use std::sync::Arc;
use std::time::Duration;
use crate::config::ServerConfig;
use crate::session::SessionStore;
use crate::templates::TemplateEngine;
use crate::services::access_logging::AccessLoggingService;
use crate::services::gc::GcService;
use crate::services::integrity::IntegrityService;
use crate::services::metrics::MetricsService;
use crate::services::replication::ReplicationManager;
use crate::services::site_registry::SiteRegistry;
use crate::services::site_sync::SiteSyncWorker;
use crate::services::system_metrics::SystemMetricsService;
use crate::services::website_domains::WebsiteDomainStore;
use crate::session::SessionStore;
use crate::stores::connections::ConnectionStore;
use crate::templates::TemplateEngine;
use myfsio_auth::iam::IamService;
use myfsio_crypto::encryption::EncryptionService;
use myfsio_crypto::kms::KmsService;
@@ -27,6 +29,7 @@ pub struct AppState {
pub gc: Option<Arc<GcService>>,
pub integrity: Option<Arc<IntegrityService>>,
pub metrics: Option<Arc<MetricsService>>,
pub system_metrics: Option<Arc<SystemMetricsService>>,
pub site_registry: Option<Arc<SiteRegistry>>,
pub website_domains: Option<Arc<WebsiteDomainStore>>,
pub connections: Arc<ConnectionStore>,
@@ -34,6 +37,7 @@ pub struct AppState {
pub site_sync: Option<Arc<SiteSyncWorker>>,
pub templates: Option<Arc<TemplateEngine>>,
pub sessions: Arc<SessionStore>,
pub access_logging: Arc<AccessLoggingService>,
}
impl AppState {
@@ -66,7 +70,23 @@ impl AppState {
let metrics = if config.metrics_enabled {
Some(Arc::new(MetricsService::new(
&config.storage_root,
crate::services::metrics::MetricsConfig::default(),
crate::services::metrics::MetricsConfig {
interval_minutes: config.metrics_interval_minutes,
retention_hours: config.metrics_retention_hours,
},
)))
} else {
None
};
let system_metrics = if config.metrics_history_enabled {
Some(Arc::new(SystemMetricsService::new(
&config.storage_root,
storage.clone(),
crate::services::system_metrics::SystemMetricsConfig {
interval_minutes: config.metrics_history_interval_minutes,
retention_hours: config.metrics_history_retention_hours,
},
)))
} else {
None
@@ -111,6 +131,7 @@ impl AppState {
};
let templates = init_templates(&config.templates_dir);
let access_logging = Arc::new(AccessLoggingService::new(&config.storage_root));
Self {
config,
storage,
@@ -120,6 +141,7 @@ impl AppState {
gc,
integrity,
metrics,
system_metrics,
site_registry,
website_domains,
connections,
@@ -127,6 +149,7 @@ impl AppState {
site_sync,
templates,
sessions: Arc::new(SessionStore::new(Duration::from_secs(60 * 60 * 12))),
access_logging,
}
}
@@ -149,9 +172,7 @@ impl AppState {
let encryption = if config.encryption_enabled {
match myfsio_crypto::kms::load_or_create_master_key(&keys_dir).await {
Ok(master_key) => {
Some(Arc::new(EncryptionService::new(master_key, kms.clone())))
}
Ok(master_key) => Some(Arc::new(EncryptionService::new(master_key, kms.clone()))),
Err(e) => {
tracing::error!("Failed to initialize encryption: {}", e);
None

95
rust/myfsio-engine/crates/myfsio-server/src/templates.rs
View File

@@ -6,7 +6,8 @@ use parking_lot::RwLock;
use serde_json::Value;
use tera::{Context, Error as TeraError, Tera};
pub type EndpointResolver = Arc<dyn Fn(&str, &HashMap<String, Value>) -> Option<String> + Send + Sync>;
pub type EndpointResolver =
Arc<dyn Fn(&str, &HashMap<String, Value>) -> Option<String> + Send + Sync>;
#[derive(Clone)]
pub struct TemplateEngine {
@@ -17,10 +18,10 @@ pub struct TemplateEngine {
impl TemplateEngine {
pub fn new(template_glob: &str) -> Result<Self, TeraError> {
let mut tera = Tera::new(template_glob)?;
tera.set_escape_fn(html_escape);
register_filters(&mut tera);
let endpoints: Arc<RwLock<HashMap<String, String>>> =
Arc::new(RwLock::new(HashMap::new()));
let endpoints: Arc<RwLock<HashMap<String, String>>> = Arc::new(RwLock::new(HashMap::new()));
register_functions(&mut tera, endpoints.clone());
@@ -52,9 +53,25 @@ impl TemplateEngine {
}
}
fn html_escape(input: &str) -> String {
let mut out = String::with_capacity(input.len());
for c in input.chars() {
match c {
'&' => out.push_str("&amp;"),
'<' => out.push_str("&lt;"),
'>' => out.push_str("&gt;"),
'"' => out.push_str("&quot;"),
'\'' => out.push_str("&#x27;"),
_ => out.push(c),
}
}
out
}
fn register_filters(tera: &mut Tera) {
tera.register_filter("format_datetime", format_datetime_filter);
tera.register_filter("filesizeformat", filesizeformat_filter);
tera.register_filter("slice", slice_filter);
}
fn register_functions(tera: &mut Tera, endpoints: Arc<RwLock<HashMap<String, String>>>) {
@@ -67,10 +84,7 @@ fn register_functions(tera: &mut Tera, endpoints: Arc<RwLock<HashMap<String, Str
.and_then(|v| v.as_str())
.ok_or_else(|| tera::Error::msg("url_for requires endpoint"))?;
if endpoint == "static" {
let filename = args
.get("filename")
.and_then(|v| v.as_str())
.unwrap_or("");
let filename = args.get("filename").and_then(|v| v.as_str()).unwrap_or("");
return Ok(Value::String(format!("/static/{}", filename)));
}
let path = match endpoints_for_url.read().get(endpoint) {
@@ -155,7 +169,11 @@ fn format_datetime_filter(value: &Value, args: &HashMap<String, Value>) -> tera:
Value::String(s) => DateTime::parse_from_rfc3339(s)
.ok()
.map(|d| d.with_timezone(&Utc))
.or_else(|| DateTime::parse_from_rfc2822(s).ok().map(|d| d.with_timezone(&Utc))),
.or_else(|| {
DateTime::parse_from_rfc2822(s)
.ok()
.map(|d| d.with_timezone(&Utc))
}),
Value::Number(n) => n.as_f64().and_then(|f| {
let secs = f as i64;
let nanos = ((f - secs as f64) * 1_000_000_000.0) as u32;
@@ -170,6 +188,51 @@ fn format_datetime_filter(value: &Value, args: &HashMap<String, Value>) -> tera:
}
}
fn slice_filter(value: &Value, args: &HashMap<String, Value>) -> tera::Result<Value> {
let start = args.get("start").and_then(|v| v.as_i64()).unwrap_or(0);
let end = args.get("end").and_then(|v| v.as_i64());
match value {
Value::String(s) => {
let chars: Vec<char> = s.chars().collect();
let len = chars.len() as i64;
let norm = |i: i64| -> usize {
if i < 0 {
(len + i).max(0) as usize
} else {
i.min(len) as usize
}
};
let s_idx = norm(start);
let e_idx = match end {
Some(e) => norm(e),
None => len as usize,
};
let e_idx = e_idx.max(s_idx);
Ok(Value::String(chars[s_idx..e_idx].iter().collect()))
}
Value::Array(arr) => {
let len = arr.len() as i64;
let norm = |i: i64| -> usize {
if i < 0 {
(len + i).max(0) as usize
} else {
i.min(len) as usize
}
};
let s_idx = norm(start);
let e_idx = match end {
Some(e) => norm(e),
None => len as usize,
};
let e_idx = e_idx.max(s_idx);
Ok(Value::Array(arr[s_idx..e_idx].to_vec()))
}
Value::Null => Ok(Value::String(String::new())),
_ => Err(tera::Error::msg("slice: unsupported value type")),
}
}
fn filesizeformat_filter(value: &Value, _args: &HashMap<String, Value>) -> tera::Result<Value> {
let bytes = match value {
Value::Number(n) => n.as_f64().unwrap_or(0.0),
@@ -205,7 +268,10 @@ mod tests {
engine.register_endpoints(&[
("ui.buckets_overview", "/ui/buckets"),
("ui.bucket_detail", "/ui/buckets/{bucket_name}"),
("ui.abort_multipart_upload", "/ui/buckets/{bucket_name}/multipart/{upload_id}/abort"),
(
"ui.abort_multipart_upload",
"/ui/buckets/{bucket_name}/multipart/{upload_id}/abort",
),
]);
engine
}
@@ -220,7 +286,10 @@ mod tests {
#[test]
fn static_url() {
let e = test_engine();
let out = render_inline(&e, "{{ url_for(endpoint='static', filename='css/main.css') }}");
let out = render_inline(
&e,
"{{ url_for(endpoint='static', filename='css/main.css') }}",
);
assert_eq!(out, "/static/css/main.css");
}
@@ -267,7 +336,11 @@ mod tests {
.get_template_names()
.map(|s| s.to_string())
.collect();
assert!(names.len() >= 10, "expected 10+ templates, got {}", names.len());
assert!(
names.len() >= 10,
"expected 10+ templates, got {}",
names.len()
);
}
#[test]

3157
rust/myfsio-engine/crates/myfsio-server/static/css/main.css Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
rust/myfsio-engine/crates/myfsio-server/static/images/MyFSIO.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 200 KiB

BIN
rust/myfsio-engine/crates/myfsio-server/static/images/MyFSIO.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 872 KiB

4827
rust/myfsio-engine/crates/myfsio-server/static/js/bucket-detail-main.js Normal file
View File

File diff suppressed because it is too large Load Diff

192
rust/myfsio-engine/crates/myfsio-server/static/js/bucket-detail-operations.js Normal file
View File

@@ -0,0 +1,192 @@
window.BucketDetailOperations = (function() {
'use strict';
let showMessage = function() {};
let escapeHtml = function(s) { return s; };
function init(config) {
showMessage = config.showMessage || showMessage;
escapeHtml = config.escapeHtml || escapeHtml;
}
async function loadLifecycleRules(card, endpoint) {
if (!card || !endpoint) return;
const body = card.querySelector('[data-lifecycle-body]');
if (!body) return;
try {
const response = await fetch(endpoint);
const data = await response.json();
if (!response.ok) {
body.innerHTML = `<tr><td colspan="5" class="text-center text-danger py-3">${escapeHtml(data.error || 'Failed to load')}</td></tr>`;
return;
}
const rules = data.rules || [];
if (rules.length === 0) {
body.innerHTML = '<tr><td colspan="5" class="text-center text-muted py-3">No lifecycle rules configured</td></tr>';
return;
}
body.innerHTML = rules.map(rule => {
const actions = [];
if (rule.expiration_days) actions.push(`Delete after ${rule.expiration_days} days`);
if (rule.noncurrent_days) actions.push(`Delete old versions after ${rule.noncurrent_days} days`);
if (rule.abort_mpu_days) actions.push(`Abort incomplete MPU after ${rule.abort_mpu_days} days`);
return `
<tr>
<td class="fw-medium">${escapeHtml(rule.id)}</td>
<td><code>${escapeHtml(rule.prefix || '(all)')}</code></td>
<td>${actions.map(a => `<div class="small">${escapeHtml(a)}</div>`).join('')}</td>
<td>
<span class="badge ${rule.status === 'Enabled' ? 'text-bg-success' : 'text-bg-secondary'}">${escapeHtml(rule.status)}</span>
</td>
<td class="text-end">
<button class="btn btn-sm btn-outline-danger" onclick="BucketDetailOperations.deleteLifecycleRule('${escapeHtml(rule.id)}')">
<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" viewBox="0 0 16 16">
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
</svg>
</button>
</td>
</tr>
`;
}).join('');
} catch (err) {
body.innerHTML = `<tr><td colspan="5" class="text-center text-danger py-3">${escapeHtml(err.message)}</td></tr>`;
}
}
async function loadCorsRules(card, endpoint) {
if (!card || !endpoint) return;
const body = document.getElementById('cors-rules-body');
if (!body) return;
try {
const response = await fetch(endpoint);
const data = await response.json();
if (!response.ok) {
body.innerHTML = `<tr><td colspan="5" class="text-center text-danger py-3">${escapeHtml(data.error || 'Failed to load')}</td></tr>`;
return;
}
const rules = data.rules || [];
if (rules.length === 0) {
body.innerHTML = '<tr><td colspan="5" class="text-center text-muted py-3">No CORS rules configured</td></tr>';
return;
}
body.innerHTML = rules.map((rule, idx) => `
<tr>
<td>${(rule.allowed_origins || []).map(o => `<code class="d-block">${escapeHtml(o)}</code>`).join('')}</td>
<td>${(rule.allowed_methods || []).map(m => `<span class="badge text-bg-secondary me-1">${escapeHtml(m)}</span>`).join('')}</td>
<td class="small text-muted">${(rule.allowed_headers || []).slice(0, 3).join(', ')}${(rule.allowed_headers || []).length > 3 ? '...' : ''}</td>
<td class="text-muted">${rule.max_age_seconds || 0}s</td>
<td class="text-end">
<button class="btn btn-sm btn-outline-danger" onclick="BucketDetailOperations.deleteCorsRule(${idx})">
<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" viewBox="0 0 16 16">
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
</svg>
</button>
</td>
</tr>
`).join('');
} catch (err) {
body.innerHTML = `<tr><td colspan="5" class="text-center text-danger py-3">${escapeHtml(err.message)}</td></tr>`;
}
}
async function loadAcl(card, endpoint) {
if (!card || !endpoint) return;
const body = card.querySelector('[data-acl-body]');
if (!body) return;
try {
const response = await fetch(endpoint);
const data = await response.json();
if (!response.ok) {
body.innerHTML = `<tr><td colspan="3" class="text-center text-danger py-3">${escapeHtml(data.error || 'Failed to load')}</td></tr>`;
return;
}
const grants = data.grants || [];
if (grants.length === 0) {
body.innerHTML = '<tr><td colspan="3" class="text-center text-muted py-3">No ACL grants configured</td></tr>';
return;
}
body.innerHTML = grants.map(grant => {
const grantee = grant.grantee_type === 'CanonicalUser'
? grant.display_name || grant.grantee_id
: grant.grantee_uri || grant.grantee_type;
return `
<tr>
<td class="fw-medium">${escapeHtml(grantee)}</td>
<td><span class="badge text-bg-info">${escapeHtml(grant.permission)}</span></td>
<td class="text-muted small">${escapeHtml(grant.grantee_type)}</td>
</tr>
`;
}).join('');
} catch (err) {
body.innerHTML = `<tr><td colspan="3" class="text-center text-danger py-3">${escapeHtml(err.message)}</td></tr>`;
}
}
async function deleteLifecycleRule(ruleId) {
if (!confirm(`Delete lifecycle rule "${ruleId}"?`)) return;
const card = document.getElementById('lifecycle-rules-card');
if (!card) return;
const endpoint = card.dataset.lifecycleUrl;
const csrfToken = window.getCsrfToken ? window.getCsrfToken() : '';
try {
const resp = await fetch(endpoint, {
method: 'DELETE',
headers: { 'Content-Type': 'application/json', 'X-CSRFToken': csrfToken },
body: JSON.stringify({ rule_id: ruleId })
});
const data = await resp.json();
if (!resp.ok) throw new Error(data.error || 'Failed to delete');
showMessage({ title: 'Rule deleted', body: `Lifecycle rule "${ruleId}" has been deleted.`, variant: 'success' });
loadLifecycleRules(card, endpoint);
} catch (err) {
showMessage({ title: 'Delete failed', body: err.message, variant: 'danger' });
}
}
async function deleteCorsRule(index) {
if (!confirm('Delete this CORS rule?')) return;
const card = document.getElementById('cors-rules-card');
if (!card) return;
const endpoint = card.dataset.corsUrl;
const csrfToken = window.getCsrfToken ? window.getCsrfToken() : '';
try {
const resp = await fetch(endpoint, {
method: 'DELETE',
headers: { 'Content-Type': 'application/json', 'X-CSRFToken': csrfToken },
body: JSON.stringify({ rule_index: index })
});
const data = await resp.json();
if (!resp.ok) throw new Error(data.error || 'Failed to delete');
showMessage({ title: 'Rule deleted', body: 'CORS rule has been deleted.', variant: 'success' });
loadCorsRules(card, endpoint);
} catch (err) {
showMessage({ title: 'Delete failed', body: err.message, variant: 'danger' });
}
}
return {
init: init,
loadLifecycleRules: loadLifecycleRules,
loadCorsRules: loadCorsRules,
loadAcl: loadAcl,
deleteLifecycleRule: deleteLifecycleRule,
deleteCorsRule: deleteCorsRule
};
})();

600
rust/myfsio-engine/crates/myfsio-server/static/js/bucket-detail-upload.js Normal file
View File

@@ -0,0 +1,600 @@
window.BucketDetailUpload = (function() {
'use strict';
const MULTIPART_THRESHOLD = 8 * 1024 * 1024;
const CHUNK_SIZE = 8 * 1024 * 1024;
const MAX_PART_RETRIES = 3;
const RETRY_BASE_DELAY_MS = 1000;
let state = {
isUploading: false,
uploadProgress: { current: 0, total: 0, currentFile: '' }
};
let elements = {};
let callbacks = {};
function init(config) {
elements = {
uploadForm: config.uploadForm,
uploadFileInput: config.uploadFileInput,
uploadModal: config.uploadModal,
uploadModalEl: config.uploadModalEl,
uploadSubmitBtn: config.uploadSubmitBtn,
uploadCancelBtn: config.uploadCancelBtn,
uploadBtnText: config.uploadBtnText,
uploadDropZone: config.uploadDropZone,
uploadDropZoneLabel: config.uploadDropZoneLabel,
uploadProgressStack: config.uploadProgressStack,
uploadKeyPrefix: config.uploadKeyPrefix,
singleFileOptions: config.singleFileOptions,
bulkUploadProgress: config.bulkUploadProgress,
bulkUploadStatus: config.bulkUploadStatus,
bulkUploadCounter: config.bulkUploadCounter,
bulkUploadProgressBar: config.bulkUploadProgressBar,
bulkUploadCurrentFile: config.bulkUploadCurrentFile,
bulkUploadResults: config.bulkUploadResults,
bulkUploadSuccessAlert: config.bulkUploadSuccessAlert,
bulkUploadErrorAlert: config.bulkUploadErrorAlert,
bulkUploadSuccessCount: config.bulkUploadSuccessCount,
bulkUploadErrorCount: config.bulkUploadErrorCount,
bulkUploadErrorList: config.bulkUploadErrorList,
floatingProgress: config.floatingProgress,
floatingProgressBar: config.floatingProgressBar,
floatingProgressStatus: config.floatingProgressStatus,
floatingProgressTitle: config.floatingProgressTitle,
floatingProgressExpand: config.floatingProgressExpand
};
callbacks = {
showMessage: config.showMessage || function() {},
formatBytes: config.formatBytes || function(b) { return b + ' bytes'; },
escapeHtml: config.escapeHtml || function(s) { return s; },
onUploadComplete: config.onUploadComplete || function() {},
hasFolders: config.hasFolders || function() { return false; },
getCurrentPrefix: config.getCurrentPrefix || function() { return ''; }
};
setupEventListeners();
setupBeforeUnload();
}
function isUploading() {
return state.isUploading;
}
function setupBeforeUnload() {
window.addEventListener('beforeunload', (e) => {
if (state.isUploading) {
e.preventDefault();
e.returnValue = 'Upload in progress. Are you sure you want to leave?';
return e.returnValue;
}
});
}
function showFloatingProgress() {
if (elements.floatingProgress) {
elements.floatingProgress.classList.remove('d-none');
}
}
function hideFloatingProgress() {
if (elements.floatingProgress) {
elements.floatingProgress.classList.add('d-none');
}
}
function updateFloatingProgress(current, total, currentFile) {
state.uploadProgress = { current, total, currentFile: currentFile || '' };
if (elements.floatingProgressBar && total > 0) {
const percent = Math.round((current / total) * 100);
elements.floatingProgressBar.style.width = `${percent}%`;
}
if (elements.floatingProgressStatus) {
if (currentFile) {
elements.floatingProgressStatus.textContent = `${current}/${total} files - ${currentFile}`;
} else {
elements.floatingProgressStatus.textContent = `${current}/${total} files completed`;
}
}
if (elements.floatingProgressTitle) {
elements.floatingProgressTitle.textContent = `Uploading ${total} file${total !== 1 ? 's' : ''}...`;
}
}
function refreshUploadDropLabel() {
if (!elements.uploadDropZoneLabel || !elements.uploadFileInput) return;
const files = elements.uploadFileInput.files;
if (!files || files.length === 0) {
elements.uploadDropZoneLabel.textContent = 'No file selected';
if (elements.singleFileOptions) elements.singleFileOptions.classList.remove('d-none');
return;
}
elements.uploadDropZoneLabel.textContent = files.length === 1 ? files[0].name : `${files.length} files selected`;
if (elements.singleFileOptions) {
elements.singleFileOptions.classList.toggle('d-none', files.length > 1);
}
}
function updateUploadBtnText() {
if (!elements.uploadBtnText || !elements.uploadFileInput) return;
const files = elements.uploadFileInput.files;
if (!files || files.length <= 1) {
elements.uploadBtnText.textContent = 'Upload';
} else {
elements.uploadBtnText.textContent = `Upload ${files.length} files`;
}
}
function resetUploadUI() {
if (elements.bulkUploadProgress) elements.bulkUploadProgress.classList.add('d-none');
if (elements.bulkUploadResults) elements.bulkUploadResults.classList.add('d-none');
if (elements.bulkUploadSuccessAlert) elements.bulkUploadSuccessAlert.classList.remove('d-none');
if (elements.bulkUploadErrorAlert) elements.bulkUploadErrorAlert.classList.add('d-none');
if (elements.bulkUploadErrorList) elements.bulkUploadErrorList.innerHTML = '';
if (elements.uploadSubmitBtn) elements.uploadSubmitBtn.disabled = false;
if (elements.uploadFileInput) elements.uploadFileInput.disabled = false;
if (elements.uploadProgressStack) elements.uploadProgressStack.innerHTML = '';
if (elements.uploadDropZone) {
elements.uploadDropZone.classList.remove('upload-locked');
elements.uploadDropZone.style.pointerEvents = '';
}
state.isUploading = false;
hideFloatingProgress();
}
function setUploadLockState(locked) {
if (elements.uploadDropZone) {
elements.uploadDropZone.classList.toggle('upload-locked', locked);
elements.uploadDropZone.style.pointerEvents = locked ? 'none' : '';
}
if (elements.uploadFileInput) {
elements.uploadFileInput.disabled = locked;
}
}
function createProgressItem(file) {
const item = document.createElement('div');
item.className = 'upload-progress-item';
item.dataset.state = 'uploading';
item.innerHTML = `
<div class="d-flex justify-content-between align-items-start">
<div class="min-width-0 flex-grow-1">
<div class="file-name">${callbacks.escapeHtml(file.name)}</div>
<div class="file-size">${callbacks.formatBytes(file.size)}</div>
</div>
<div class="upload-status text-end ms-2">Preparing...</div>
</div>
<div class="progress-container">
<div class="progress">
<div class="progress-bar bg-primary" role="progressbar" style="width: 0%"></div>
</div>
<div class="progress-text">
<span class="progress-loaded">0 B</span>
<span class="progress-percent">0%</span>
</div>
</div>
`;
return item;
}
function updateProgressItem(item, { loaded, total, status, progressState, error }) {
if (progressState) item.dataset.state = progressState;
const statusEl = item.querySelector('.upload-status');
const progressBar = item.querySelector('.progress-bar');
const progressLoaded = item.querySelector('.progress-loaded');
const progressPercent = item.querySelector('.progress-percent');
if (status) {
statusEl.textContent = status;
statusEl.className = 'upload-status text-end ms-2';
if (progressState === 'success') statusEl.classList.add('success');
if (progressState === 'error') statusEl.classList.add('error');
}
if (typeof loaded === 'number' && typeof total === 'number' && total > 0) {
const percent = Math.round((loaded / total) * 100);
progressBar.style.width = `${percent}%`;
progressLoaded.textContent = `${callbacks.formatBytes(loaded)} / ${callbacks.formatBytes(total)}`;
progressPercent.textContent = `${percent}%`;
}
if (error) {
const progressContainer = item.querySelector('.progress-container');
if (progressContainer) {
progressContainer.innerHTML = `<div class="text-danger small mt-1">${callbacks.escapeHtml(error)}</div>`;
}
}
}
function uploadPartXHR(url, chunk, csrfToken, baseBytes, fileSize, progressItem, partNumber, totalParts) {
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open('PUT', url, true);
xhr.setRequestHeader('X-CSRFToken', csrfToken || '');
xhr.upload.addEventListener('progress', (e) => {
if (e.lengthComputable) {
updateProgressItem(progressItem, {
status: `Part ${partNumber}/${totalParts}`,
loaded: baseBytes + e.loaded,
total: fileSize
});
}
});
xhr.addEventListener('load', () => {
if (xhr.status >= 200 && xhr.status < 300) {
try {
resolve(JSON.parse(xhr.responseText));
} catch {
reject(new Error(`Part ${partNumber}: invalid response`));
}
} else {
try {
const data = JSON.parse(xhr.responseText);
reject(new Error(data.error || `Part ${partNumber} failed (${xhr.status})`));
} catch {
reject(new Error(`Part ${partNumber} failed (${xhr.status})`));
}
}
});
xhr.addEventListener('error', () => reject(new Error(`Part ${partNumber}: network error`)));
xhr.addEventListener('abort', () => reject(new Error(`Part ${partNumber}: aborted`)));
xhr.send(chunk);
});
}
async function uploadPartWithRetry(url, chunk, csrfToken, baseBytes, fileSize, progressItem, partNumber, totalParts) {
let lastError;
for (let attempt = 0; attempt <= MAX_PART_RETRIES; attempt++) {
try {
return await uploadPartXHR(url, chunk, csrfToken, baseBytes, fileSize, progressItem, partNumber, totalParts);
} catch (err) {
lastError = err;
if (attempt < MAX_PART_RETRIES) {
const delay = RETRY_BASE_DELAY_MS * Math.pow(2, attempt);
updateProgressItem(progressItem, {
status: `Part ${partNumber}/${totalParts} retry ${attempt + 1}/${MAX_PART_RETRIES}...`,
loaded: baseBytes,
total: fileSize
});
await new Promise(r => setTimeout(r, delay));
}
}
}
throw lastError;
}
async function uploadMultipart(file, objectKey, metadata, progressItem, urls) {
const csrfToken = document.querySelector('input[name="csrf_token"]')?.value;
updateProgressItem(progressItem, { status: 'Initiating...', loaded: 0, total: file.size });
const initResp = await fetch(urls.initUrl, {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'X-CSRFToken': csrfToken || '' },
body: JSON.stringify({ object_key: objectKey, metadata })
});
if (!initResp.ok) {
const err = await initResp.json().catch(() => ({}));
throw new Error(err.error || 'Failed to initiate upload');
}
const { upload_id } = await initResp.json();
const partUrl = urls.partTemplate.replace('UPLOAD_ID_PLACEHOLDER', upload_id);
const completeUrl = urls.completeTemplate.replace('UPLOAD_ID_PLACEHOLDER', upload_id);
const abortUrl = urls.abortTemplate.replace('UPLOAD_ID_PLACEHOLDER', upload_id);
const parts = [];
const totalParts = Math.ceil(file.size / CHUNK_SIZE);
let uploadedBytes = 0;
try {
for (let partNumber = 1; partNumber <= totalParts; partNumber++) {
const start = (partNumber - 1) * CHUNK_SIZE;
const end = Math.min(start + CHUNK_SIZE, file.size);
const chunk = file.slice(start, end);
const partData = await uploadPartWithRetry(
`${partUrl}?partNumber=${partNumber}`,
chunk, csrfToken, uploadedBytes, file.size,
progressItem, partNumber, totalParts
);
parts.push({ part_number: partNumber, etag: partData.etag });
uploadedBytes += (end - start);
updateProgressItem(progressItem, {
loaded: uploadedBytes,
total: file.size
});
}
updateProgressItem(progressItem, { status: 'Completing...', loaded: file.size, total: file.size });
const completeResp = await fetch(completeUrl, {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'X-CSRFToken': csrfToken || '' },
body: JSON.stringify({ parts })
});
if (!completeResp.ok) {
const err = await completeResp.json().catch(() => ({}));
throw new Error(err.error || 'Failed to complete upload');
}
return await completeResp.json();
} catch (err) {
try {
await fetch(abortUrl, { method: 'DELETE', headers: { 'X-CSRFToken': csrfToken || '' } });
} catch {}
throw err;
}
}
async function uploadRegular(file, objectKey, metadata, progressItem, formAction) {
return new Promise((resolve, reject) => {
const formData = new FormData();
formData.append('object', file);
formData.append('object_key', objectKey);
if (metadata) formData.append('metadata', JSON.stringify(metadata));
const csrfToken = document.querySelector('input[name="csrf_token"]')?.value;
if (csrfToken) formData.append('csrf_token', csrfToken);
const xhr = new XMLHttpRequest();
xhr.open('POST', formAction, true);
xhr.setRequestHeader('X-Requested-With', 'XMLHttpRequest');
xhr.setRequestHeader('X-CSRFToken', csrfToken || '');
xhr.upload.addEventListener('progress', (e) => {
if (e.lengthComputable) {
updateProgressItem(progressItem, {
status: 'Uploading...',
loaded: e.loaded,
total: e.total
});
}
});
xhr.addEventListener('load', () => {
if (xhr.status >= 200 && xhr.status < 300) {
try {
const data = JSON.parse(xhr.responseText);
if (data.status === 'error') {
reject(new Error(data.message || 'Upload failed'));
} else {
resolve(data);
}
} catch {
resolve({});
}
} else {
try {
const data = JSON.parse(xhr.responseText);
reject(new Error(data.message || `Upload failed (${xhr.status})`));
} catch {
reject(new Error(`Upload failed (${xhr.status})`));
}
}
});
xhr.addEventListener('error', () => reject(new Error('Network error')));
xhr.addEventListener('abort', () => reject(new Error('Upload aborted')));
xhr.send(formData);
});
}
async function uploadSingleFile(file, keyPrefix, metadata, progressItem, urls) {
const objectKey = keyPrefix ? `${keyPrefix}${file.name}` : file.name;
const shouldUseMultipart = file.size >= MULTIPART_THRESHOLD && urls.initUrl;
if (!progressItem && elements.uploadProgressStack) {
progressItem = createProgressItem(file);
elements.uploadProgressStack.appendChild(progressItem);
}
try {
let result;
if (shouldUseMultipart) {
updateProgressItem(progressItem, { status: 'Multipart upload...', loaded: 0, total: file.size });
result = await uploadMultipart(file, objectKey, metadata, progressItem, urls);
} else {
updateProgressItem(progressItem, { status: 'Uploading...', loaded: 0, total: file.size });
result = await uploadRegular(file, objectKey, metadata, progressItem, urls.formAction);
}
updateProgressItem(progressItem, { progressState: 'success', status: 'Complete', loaded: file.size, total: file.size });
return result;
} catch (err) {
updateProgressItem(progressItem, { progressState: 'error', status: 'Failed', error: err.message });
throw err;
}
}
async function performBulkUpload(files, urls) {
if (state.isUploading || !files || files.length === 0) return;
state.isUploading = true;
setUploadLockState(true);
const keyPrefix = (elements.uploadKeyPrefix?.value || '').trim();
const metadataRaw = elements.uploadForm?.querySelector('textarea[name="metadata"]')?.value?.trim();
let metadata = null;
if (metadataRaw) {
try {
metadata = JSON.parse(metadataRaw);
} catch {
callbacks.showMessage({ title: 'Invalid metadata', body: 'Metadata must be valid JSON.', variant: 'danger' });
resetUploadUI();
return;
}
}
if (elements.bulkUploadProgress) elements.bulkUploadProgress.classList.remove('d-none');
if (elements.bulkUploadResults) elements.bulkUploadResults.classList.add('d-none');
if (elements.uploadSubmitBtn) elements.uploadSubmitBtn.disabled = true;
if (elements.uploadFileInput) elements.uploadFileInput.disabled = true;
const successFiles = [];
const errorFiles = [];
const total = files.length;
updateFloatingProgress(0, total, files[0]?.name || '');
for (let i = 0; i < total; i++) {
const file = files[i];
const current = i + 1;
if (elements.bulkUploadCounter) elements.bulkUploadCounter.textContent = `${current}/${total}`;
if (elements.bulkUploadCurrentFile) elements.bulkUploadCurrentFile.textContent = `Uploading: ${file.name}`;
if (elements.bulkUploadProgressBar) {
const percent = Math.round((current / total) * 100);
elements.bulkUploadProgressBar.style.width = `${percent}%`;
}
updateFloatingProgress(i, total, file.name);
try {
await uploadSingleFile(file, keyPrefix, metadata, null, urls);
successFiles.push(file.name);
} catch (error) {
errorFiles.push({ name: file.name, error: error.message || 'Unknown error' });
}
}
updateFloatingProgress(total, total);
if (elements.bulkUploadProgress) elements.bulkUploadProgress.classList.add('d-none');
if (elements.bulkUploadResults) elements.bulkUploadResults.classList.remove('d-none');
if (elements.bulkUploadSuccessCount) elements.bulkUploadSuccessCount.textContent = successFiles.length;
if (successFiles.length === 0 && elements.bulkUploadSuccessAlert) {
elements.bulkUploadSuccessAlert.classList.add('d-none');
}
if (errorFiles.length > 0) {
if (elements.bulkUploadErrorCount) elements.bulkUploadErrorCount.textContent = errorFiles.length;
if (elements.bulkUploadErrorAlert) elements.bulkUploadErrorAlert.classList.remove('d-none');
if (elements.bulkUploadErrorList) {
elements.bulkUploadErrorList.innerHTML = errorFiles
.map(f => `<li><strong>${callbacks.escapeHtml(f.name)}</strong>: ${callbacks.escapeHtml(f.error)}</li>`)
.join('');
}
}
state.isUploading = false;
setUploadLockState(false);
if (successFiles.length > 0) {
if (elements.uploadBtnText) elements.uploadBtnText.textContent = 'Refreshing...';
callbacks.onUploadComplete(successFiles, errorFiles);
} else {
if (elements.uploadSubmitBtn) elements.uploadSubmitBtn.disabled = false;
if (elements.uploadFileInput) elements.uploadFileInput.disabled = false;
}
}
function setupEventListeners() {
if (elements.uploadFileInput) {
elements.uploadFileInput.addEventListener('change', () => {
if (state.isUploading) return;
refreshUploadDropLabel();
updateUploadBtnText();
resetUploadUI();
});
}
if (elements.uploadDropZone) {
elements.uploadDropZone.addEventListener('click', () => {
if (state.isUploading) return;
elements.uploadFileInput?.click();
});
}
if (elements.floatingProgressExpand) {
elements.floatingProgressExpand.addEventListener('click', () => {
if (elements.uploadModal) {
elements.uploadModal.show();
}
});
}
if (elements.uploadModalEl) {
elements.uploadModalEl.addEventListener('hide.bs.modal', () => {
if (state.isUploading) {
showFloatingProgress();
}
});
elements.uploadModalEl.addEventListener('hidden.bs.modal', () => {
if (!state.isUploading) {
resetUploadUI();
if (elements.uploadFileInput) elements.uploadFileInput.value = '';
refreshUploadDropLabel();
updateUploadBtnText();
}
});
elements.uploadModalEl.addEventListener('show.bs.modal', () => {
if (state.isUploading) {
hideFloatingProgress();
}
if (callbacks.hasFolders() && callbacks.getCurrentPrefix()) {
if (elements.uploadKeyPrefix) {
elements.uploadKeyPrefix.value = callbacks.getCurrentPrefix();
}
} else if (elements.uploadKeyPrefix) {
elements.uploadKeyPrefix.value = '';
}
});
}
}
function wireDropTarget(target, options) {
const { highlightClass = '', autoOpenModal = false } = options || {};
if (!target) return;
const preventDefaults = (event) => {
event.preventDefault();
event.stopPropagation();
};
['dragenter', 'dragover'].forEach((eventName) => {
target.addEventListener(eventName, (event) => {
preventDefaults(event);
if (state.isUploading) return;
if (highlightClass) {
target.classList.add(highlightClass);
}
});
});
['dragleave', 'drop'].forEach((eventName) => {
target.addEventListener(eventName, (event) => {
preventDefaults(event);
if (highlightClass) {
target.classList.remove(highlightClass);
}
});
});
target.addEventListener('drop', (event) => {
if (state.isUploading) return;
if (!event.dataTransfer?.files?.length || !elements.uploadFileInput) {
return;
}
elements.uploadFileInput.files = event.dataTransfer.files;
elements.uploadFileInput.dispatchEvent(new Event('change', { bubbles: true }));
if (autoOpenModal && elements.uploadModal) {
elements.uploadModal.show();
}
});
}
return {
init: init,
isUploading: isUploading,
performBulkUpload: performBulkUpload,
wireDropTarget: wireDropTarget,
resetUploadUI: resetUploadUI,
refreshUploadDropLabel: refreshUploadDropLabel,
updateUploadBtnText: updateUploadBtnText
};
})();

120
rust/myfsio-engine/crates/myfsio-server/static/js/bucket-detail-utils.js Normal file
View File

@@ -0,0 +1,120 @@
window.BucketDetailUtils = (function() {
'use strict';
function setupJsonAutoIndent(textarea) {
if (!textarea) return;
textarea.addEventListener('keydown', function(e) {
if (e.key === 'Enter') {
e.preventDefault();
const start = this.selectionStart;
const end = this.selectionEnd;
const value = this.value;
const lineStart = value.lastIndexOf('\n', start - 1) + 1;
const currentLine = value.substring(lineStart, start);
const indentMatch = currentLine.match(/^(\s*)/);
let indent = indentMatch ? indentMatch[1] : '';
const trimmedLine = currentLine.trim();
const lastChar = trimmedLine.slice(-1);
let newIndent = indent;
let insertAfter = '';
if (lastChar === '{' || lastChar === '[') {
newIndent = indent + ' ';
const charAfterCursor = value.substring(start, start + 1).trim();
if ((lastChar === '{' && charAfterCursor === '}') ||
(lastChar === '[' && charAfterCursor === ']')) {
insertAfter = '\n' + indent;
}
} else if (lastChar === ',' || lastChar === ':') {
newIndent = indent;
}
const insertion = '\n' + newIndent + insertAfter;
const newValue = value.substring(0, start) + insertion + value.substring(end);
this.value = newValue;
const newCursorPos = start + 1 + newIndent.length;
this.selectionStart = this.selectionEnd = newCursorPos;
this.dispatchEvent(new Event('input', { bubbles: true }));
}
if (e.key === 'Tab') {
e.preventDefault();
const start = this.selectionStart;
const end = this.selectionEnd;
if (e.shiftKey) {
const lineStart = this.value.lastIndexOf('\n', start - 1) + 1;
const lineContent = this.value.substring(lineStart, start);
if (lineContent.startsWith(' ')) {
this.value = this.value.substring(0, lineStart) +
this.value.substring(lineStart + 2);
this.selectionStart = this.selectionEnd = Math.max(lineStart, start - 2);
}
} else {
this.value = this.value.substring(0, start) + ' ' + this.value.substring(end);
this.selectionStart = this.selectionEnd = start + 2;
}
this.dispatchEvent(new Event('input', { bubbles: true }));
}
});
}
function formatBytes(bytes) {
if (!Number.isFinite(bytes)) return `${bytes} bytes`;
const units = ['bytes', 'KB', 'MB', 'GB', 'TB'];
let i = 0;
let size = bytes;
while (size >= 1024 && i < units.length - 1) {
size /= 1024;
i++;
}
return `${size.toFixed(i === 0 ? 0 : 1)} ${units[i]}`;
}
function escapeHtml(value) {
if (value === null || value === undefined) return '';
return String(value)
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#039;');
}
function fallbackCopy(text) {
const textArea = document.createElement('textarea');
textArea.value = text;
textArea.style.position = 'fixed';
textArea.style.left = '-9999px';
textArea.style.top = '-9999px';
document.body.appendChild(textArea);
textArea.focus();
textArea.select();
let success = false;
try {
success = document.execCommand('copy');
} catch {
success = false;
}
document.body.removeChild(textArea);
return success;
}
return {
setupJsonAutoIndent: setupJsonAutoIndent,
formatBytes: formatBytes,
escapeHtml: escapeHtml,
fallbackCopy: fallbackCopy
};
})();

343
rust/myfsio-engine/crates/myfsio-server/static/js/connections-management.js Normal file
View File

@@ -0,0 +1,343 @@
window.ConnectionsManagement = (function() {
'use strict';
var endpoints = {};
var csrfToken = '';
function init(config) {
endpoints = config.endpoints || {};
csrfToken = config.csrfToken || '';
setupEventListeners();
checkAllConnectionHealth();
}
function togglePassword(id) {
var input = document.getElementById(id);
if (input) {
input.type = input.type === 'password' ? 'text' : 'password';
}
}
async function testConnection(formId, resultId) {
var form = document.getElementById(formId);
var resultDiv = document.getElementById(resultId);
if (!form || !resultDiv) return;
var formData = new FormData(form);
var data = {};
formData.forEach(function(value, key) {
if (key !== 'csrf_token') {
data[key] = value;
}
});
resultDiv.innerHTML = '<div class="text-info"><span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span> Testing connection...</div>';
var controller = new AbortController();
var timeoutId = setTimeout(function() { controller.abort(); }, 20000);
try {
var response = await fetch(endpoints.test, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-CSRFToken': csrfToken
},
body: JSON.stringify(data),
signal: controller.signal
});
clearTimeout(timeoutId);
var result = await response.json();
if (response.ok) {
resultDiv.innerHTML = '<div class="text-success">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16">' +
'<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z"/>' +
'</svg>' + window.UICore.escapeHtml(result.message) + '</div>';
} else {
resultDiv.innerHTML = '<div class="text-danger">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16">' +
'<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/>' +
'</svg>' + window.UICore.escapeHtml(result.message) + '</div>';
}
} catch (error) {
clearTimeout(timeoutId);
var message = error.name === 'AbortError'
? 'Connection test timed out - endpoint may be unreachable'
: 'Connection failed: Network error';
resultDiv.innerHTML = '<div class="text-danger">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16">' +
'<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/>' +
'</svg>' + message + '</div>';
}
}
async function checkConnectionHealth(connectionId, statusEl) {
if (!statusEl) return;
try {
var controller = new AbortController();
var timeoutId = setTimeout(function() { controller.abort(); }, 10000);
var response = await fetch(endpoints.healthTemplate.replace('CONNECTION_ID', connectionId), {
signal: controller.signal
});
clearTimeout(timeoutId);
var data = await response.json();
if (data.healthy) {
statusEl.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-success" viewBox="0 0 16 16">' +
'<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z"/></svg>';
statusEl.setAttribute('data-status', 'healthy');
statusEl.setAttribute('title', 'Connected');
} else {
statusEl.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-danger" viewBox="0 0 16 16">' +
'<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/></svg>';
statusEl.setAttribute('data-status', 'unhealthy');
statusEl.setAttribute('title', data.error || 'Unreachable');
}
} catch (error) {
statusEl.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-warning" viewBox="0 0 16 16">' +
'<path d="M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z"/></svg>';
statusEl.setAttribute('data-status', 'unknown');
statusEl.setAttribute('title', 'Could not check status');
}
}
function checkAllConnectionHealth() {
var rows = document.querySelectorAll('tr[data-connection-id]');
rows.forEach(function(row, index) {
var connectionId = row.getAttribute('data-connection-id');
var statusEl = row.querySelector('.connection-status');
if (statusEl) {
setTimeout(function() {
checkConnectionHealth(connectionId, statusEl);
}, index * 200);
}
});
}
function updateConnectionCount() {
var countBadge = document.querySelector('.badge.bg-primary.bg-opacity-10.text-primary.fs-6');
if (countBadge) {
var remaining = document.querySelectorAll('tr[data-connection-id]').length;
countBadge.textContent = remaining + ' connection' + (remaining !== 1 ? 's' : '');
}
}
function createConnectionRowHtml(conn) {
var ak = conn.access_key || '';
var maskedKey = ak.length > 12 ? ak.slice(0, 8) + '...' + ak.slice(-4) : ak;
return '<tr data-connection-id="' + window.UICore.escapeHtml(conn.id) + '">' +
'<td class="text-center">' +
'<span class="connection-status" data-status="checking" title="Checking...">' +
'<span class="spinner-border spinner-border-sm text-muted" role="status" style="width: 12px; height: 12px;"></span>' +
'</span></td>' +
'<td><div class="d-flex align-items-center gap-2">' +
'<div class="connection-icon"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16">' +
'<path d="M4.406 3.342A5.53 5.53 0 0 1 8 2c2.69 0 4.923 2 5.166 4.579C14.758 6.804 16 8.137 16 9.773 16 11.569 14.502 13 12.687 13H3.781C1.708 13 0 11.366 0 9.318c0-1.763 1.266-3.223 2.942-3.593.143-.863.698-1.723 1.464-2.383z"/></svg></div>' +
'<span class="fw-medium">' + window.UICore.escapeHtml(conn.name) + '</span>' +
'</div></td>' +
'<td><span class="text-muted small text-truncate d-inline-block" style="max-width: 200px;" title="' + window.UICore.escapeHtml(conn.endpoint_url) + '">' + window.UICore.escapeHtml(conn.endpoint_url) + '</span></td>' +
'<td><span class="badge bg-primary bg-opacity-10 text-primary">' + window.UICore.escapeHtml(conn.region) + '</span></td>' +
'<td><code class="small">' + window.UICore.escapeHtml(maskedKey) + '</code></td>' +
'<td class="text-end"><div class="btn-group btn-group-sm" role="group">' +
'<button type="button" class="btn btn-outline-secondary" data-bs-toggle="modal" data-bs-target="#editConnectionModal" ' +
'data-id="' + window.UICore.escapeHtml(conn.id) + '" data-name="' + window.UICore.escapeHtml(conn.name) + '" ' +
'data-endpoint="' + window.UICore.escapeHtml(conn.endpoint_url) + '" data-region="' + window.UICore.escapeHtml(conn.region) + '" ' +
'data-access="' + window.UICore.escapeHtml(conn.access_key) + '" title="Edit connection">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">' +
'<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/></svg></button>' +
'<button type="button" class="btn btn-outline-danger" data-bs-toggle="modal" data-bs-target="#deleteConnectionModal" ' +
'data-id="' + window.UICore.escapeHtml(conn.id) + '" data-name="' + window.UICore.escapeHtml(conn.name) + '" title="Delete connection">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">' +
'<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>' +
'<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/></svg></button>' +
'</div></td></tr>';
}
function setupEventListeners() {
var testBtn = document.getElementById('testConnectionBtn');
if (testBtn) {
testBtn.addEventListener('click', function() {
testConnection('createConnectionForm', 'testResult');
});
}
var editTestBtn = document.getElementById('editTestConnectionBtn');
if (editTestBtn) {
editTestBtn.addEventListener('click', function() {
testConnection('editConnectionForm', 'editTestResult');
});
}
var editModal = document.getElementById('editConnectionModal');
if (editModal) {
editModal.addEventListener('show.bs.modal', function(event) {
var button = event.relatedTarget;
if (!button) return;
var id = button.getAttribute('data-id');
document.getElementById('edit_name').value = button.getAttribute('data-name') || '';
document.getElementById('edit_endpoint_url').value = button.getAttribute('data-endpoint') || '';
document.getElementById('edit_region').value = button.getAttribute('data-region') || '';
document.getElementById('edit_access_key').value = button.getAttribute('data-access') || '';
document.getElementById('edit_secret_key').value = '';
document.getElementById('edit_secret_key').placeholder = '(unchanged — leave blank to keep current)';
document.getElementById('edit_secret_key').required = false;
document.getElementById('editTestResult').innerHTML = '';
var form = document.getElementById('editConnectionForm');
form.action = endpoints.updateTemplate.replace('CONNECTION_ID', id);
});
}
var deleteModal = document.getElementById('deleteConnectionModal');
if (deleteModal) {
deleteModal.addEventListener('show.bs.modal', function(event) {
var button = event.relatedTarget;
if (!button) return;
var id = button.getAttribute('data-id');
var name = button.getAttribute('data-name');
document.getElementById('deleteConnectionName').textContent = name;
var form = document.getElementById('deleteConnectionForm');
form.action = endpoints.deleteTemplate.replace('CONNECTION_ID', id);
});
}
var createForm = document.getElementById('createConnectionForm');
if (createForm) {
createForm.addEventListener('submit', function(e) {
e.preventDefault();
window.UICore.submitFormAjax(createForm, {
successMessage: 'Connection created',
onSuccess: function(data) {
createForm.reset();
document.getElementById('testResult').innerHTML = '';
if (data.connection) {
var emptyState = document.querySelector('.empty-state');
if (emptyState) {
var cardBody = emptyState.closest('.card-body');
if (cardBody) {
cardBody.innerHTML = '<div class="table-responsive"><table class="table table-hover align-middle mb-0">' +
'<thead class="table-light"><tr>' +
'<th scope="col" style="width: 50px;">Status</th>' +
'<th scope="col">Name</th><th scope="col">Endpoint</th>' +
'<th scope="col">Region</th><th scope="col">Access Key</th>' +
'<th scope="col" class="text-end">Actions</th></tr></thead>' +
'<tbody></tbody></table></div>';
}
}
var tbody = document.querySelector('table tbody');
if (tbody) {
tbody.insertAdjacentHTML('beforeend', createConnectionRowHtml(data.connection));
var newRow = tbody.lastElementChild;
var statusEl = newRow.querySelector('.connection-status');
if (statusEl) {
checkConnectionHealth(data.connection.id, statusEl);
}
}
updateConnectionCount();
} else {
location.reload();
}
}
});
});
}
var editForm = document.getElementById('editConnectionForm');
if (editForm) {
editForm.addEventListener('submit', function(e) {
e.preventDefault();
window.UICore.submitFormAjax(editForm, {
successMessage: 'Connection updated',
onSuccess: function(data) {
var modal = bootstrap.Modal.getInstance(document.getElementById('editConnectionModal'));
if (modal) modal.hide();
var connId = editForm.action.split('/').slice(-2)[0];
var row = document.querySelector('tr[data-connection-id="' + connId + '"]');
if (row && data.connection) {
var nameCell = row.querySelector('.fw-medium');
if (nameCell) nameCell.textContent = data.connection.name;
var endpointCell = row.querySelector('.text-truncate');
if (endpointCell) {
endpointCell.textContent = data.connection.endpoint_url;
endpointCell.title = data.connection.endpoint_url;
}
var regionBadge = row.querySelector('.badge.bg-primary');
if (regionBadge) regionBadge.textContent = data.connection.region;
var accessCode = row.querySelector('code.small');
if (accessCode && data.connection.access_key) {
var ak = data.connection.access_key;
accessCode.textContent = ak.slice(0, 8) + '...' + ak.slice(-4);
}
var editBtn = row.querySelector('[data-bs-target="#editConnectionModal"]');
if (editBtn) {
editBtn.setAttribute('data-name', data.connection.name);
editBtn.setAttribute('data-endpoint', data.connection.endpoint_url);
editBtn.setAttribute('data-region', data.connection.region);
editBtn.setAttribute('data-access', data.connection.access_key);
}
var deleteBtn = row.querySelector('[data-bs-target="#deleteConnectionModal"]');
if (deleteBtn) {
deleteBtn.setAttribute('data-name', data.connection.name);
}
var statusEl = row.querySelector('.connection-status');
if (statusEl) {
checkConnectionHealth(connId, statusEl);
}
}
}
});
});
}
var deleteForm = document.getElementById('deleteConnectionForm');
if (deleteForm) {
deleteForm.addEventListener('submit', function(e) {
e.preventDefault();
window.UICore.submitFormAjax(deleteForm, {
successMessage: 'Connection deleted',
onSuccess: function(data) {
var modal = bootstrap.Modal.getInstance(document.getElementById('deleteConnectionModal'));
if (modal) modal.hide();
var connId = deleteForm.action.split('/').slice(-2)[0];
var row = document.querySelector('tr[data-connection-id="' + connId + '"]');
if (row) {
row.remove();
}
updateConnectionCount();
if (document.querySelectorAll('tr[data-connection-id]').length === 0) {
location.reload();
}
}
});
});
}
}
return {
init: init,
togglePassword: togglePassword,
testConnection: testConnection,
checkConnectionHealth: checkConnectionHealth
};
})();

846
rust/myfsio-engine/crates/myfsio-server/static/js/iam-management.js Normal file
View File

@@ -0,0 +1,846 @@
window.IAMManagement = (function() {
'use strict';
var users = [];
var currentUserKey = null;
var endpoints = {};
var csrfToken = '';
var iamLocked = false;
var policyModal = null;
var editUserModal = null;
var deleteUserModal = null;
var rotateSecretModal = null;
var expiryModal = null;
var currentRotateKey = null;
var currentEditKey = null;
var currentDeleteKey = null;
var currentEditAccessKey = null;
var currentDeleteAccessKey = null;
var currentExpiryKey = null;
var currentExpiryAccessKey = null;
var ALL_S3_ACTIONS = [
'list', 'read', 'write', 'delete', 'share', 'policy',
'replication', 'lifecycle', 'cors',
'create_bucket', 'delete_bucket',
'versioning', 'tagging', 'encryption', 'quota',
'object_lock', 'notification', 'logging', 'website'
];
var policyTemplates = {
full: [{ bucket: '*', actions: ['list', 'read', 'write', 'delete', 'share', 'policy', 'create_bucket', 'delete_bucket', 'replication', 'lifecycle', 'cors', 'versioning', 'tagging', 'encryption', 'quota', 'object_lock', 'notification', 'logging', 'website', 'iam:*'] }],
readonly: [{ bucket: '*', actions: ['list', 'read'] }],
writer: [{ bucket: '*', actions: ['list', 'read', 'write'] }],
operator: [{ bucket: '*', actions: ['list', 'read', 'write', 'delete', 'create_bucket', 'delete_bucket'] }],
bucketadmin: [{ bucket: '*', actions: ['list', 'read', 'write', 'delete', 'share', 'policy', 'create_bucket', 'delete_bucket', 'versioning', 'tagging', 'encryption', 'cors', 'lifecycle', 'quota', 'object_lock', 'notification', 'logging', 'website', 'replication'] }]
};
function isAdminUser(policies) {
if (!policies || !policies.length) return false;
return policies.some(function(p) {
return p.actions && (p.actions.indexOf('iam:*') >= 0 || p.actions.indexOf('*') >= 0);
});
}
function getPermissionLevel(actions) {
if (!actions || !actions.length) return 'Custom (0)';
if (actions.indexOf('*') >= 0) return 'Full Access';
if (actions.length >= ALL_S3_ACTIONS.length) {
var hasAll = ALL_S3_ACTIONS.every(function(a) { return actions.indexOf(a) >= 0; });
if (hasAll) return 'Full Access';
}
var has = function(a) { return actions.indexOf(a) >= 0; };
if (has('list') && has('read') && has('write') && has('delete')) return 'Read + Write + Delete';
if (has('list') && has('read') && has('write')) return 'Read + Write';
if (has('list') && has('read')) return 'Read Only';
return 'Custom (' + actions.length + ')';
}
function getBucketLabel(bucket) {
return bucket === '*' ? 'All Buckets' : bucket;
}
function buildUserUrl(template, userId) {
return template.replace('USER_ID', encodeURIComponent(userId));
}
function getUserByIdentifier(identifier) {
return users.find(function(u) {
return u.user_id === identifier || u.access_key === identifier;
}) || null;
}
function getUserById(userId) {
return users.find(function(u) { return u.user_id === userId; }) || null;
}
function init(config) {
users = config.users || [];
currentUserKey = config.currentUserKey || null;
endpoints = config.endpoints || {};
csrfToken = config.csrfToken || '';
iamLocked = config.iamLocked || false;
if (iamLocked) return;
initModals();
setupJsonAutoIndent();
setupCopyButtons();
setupPolicyEditor();
setupCreateUserModal();
setupEditUserModal();
setupDeleteUserModal();
setupRotateSecretModal();
setupExpiryModal();
setupFormHandlers();
setupSearch();
setupCopyAccessKeyButtons();
}
function initModals() {
var policyModalEl = document.getElementById('policyEditorModal');
var editModalEl = document.getElementById('editUserModal');
var deleteModalEl = document.getElementById('deleteUserModal');
var rotateModalEl = document.getElementById('rotateSecretModal');
var expiryModalEl = document.getElementById('expiryModal');
if (policyModalEl) policyModal = new bootstrap.Modal(policyModalEl);
if (editModalEl) editUserModal = new bootstrap.Modal(editModalEl);
if (deleteModalEl) deleteUserModal = new bootstrap.Modal(deleteModalEl);
if (rotateModalEl) rotateSecretModal = new bootstrap.Modal(rotateModalEl);
if (expiryModalEl) expiryModal = new bootstrap.Modal(expiryModalEl);
}
function setupJsonAutoIndent() {
window.UICore.setupJsonAutoIndent(document.getElementById('policyEditorDocument'));
window.UICore.setupJsonAutoIndent(document.getElementById('createUserPolicies'));
}
function setupCopyButtons() {
document.querySelectorAll('.config-copy').forEach(function(button) {
button.addEventListener('click', async function() {
var targetId = button.dataset.copyTarget;
var target = document.getElementById(targetId);
if (!target) return;
await window.UICore.copyToClipboard(target.innerText, button, 'Copy JSON');
});
});
var accessKeyCopyButton = document.querySelector('[data-access-key-copy]');
if (accessKeyCopyButton) {
accessKeyCopyButton.addEventListener('click', async function() {
var accessKeyInput = document.getElementById('disclosedAccessKeyValue');
if (!accessKeyInput) return;
await window.UICore.copyToClipboard(accessKeyInput.value, accessKeyCopyButton, 'Copy');
});
}
var secretCopyButton = document.querySelector('[data-secret-copy]');
if (secretCopyButton) {
secretCopyButton.addEventListener('click', async function() {
var secretInput = document.getElementById('disclosedSecretValue');
if (!secretInput) return;
await window.UICore.copyToClipboard(secretInput.value, secretCopyButton, 'Copy');
});
}
}
function getUserPolicies(identifier) {
var user = getUserByIdentifier(identifier);
return user ? JSON.stringify(user.policies, null, 2) : '';
}
function applyPolicyTemplate(name, textareaEl) {
if (policyTemplates[name] && textareaEl) {
textareaEl.value = JSON.stringify(policyTemplates[name], null, 2);
}
}
function setupPolicyEditor() {
var userLabelEl = document.getElementById('policyEditorUserLabel');
var userInputEl = document.getElementById('policyEditorUserId');
var textareaEl = document.getElementById('policyEditorDocument');
document.querySelectorAll('[data-policy-template]').forEach(function(button) {
button.addEventListener('click', function() {
applyPolicyTemplate(button.dataset.policyTemplate, textareaEl);
});
});
document.querySelectorAll('[data-policy-editor]').forEach(function(button) {
button.addEventListener('click', function() {
var userId = button.dataset.userId;
var accessKey = button.dataset.accessKey || userId;
if (!userId) return;
userLabelEl.textContent = accessKey;
userInputEl.value = userId;
textareaEl.value = getUserPolicies(userId);
policyModal.show();
});
});
}
function generateSecureHex(byteCount) {
var arr = new Uint8Array(byteCount);
crypto.getRandomValues(arr);
return Array.from(arr).map(function(b) { return b.toString(16).padStart(2, '0'); }).join('');
}
function generateSecureBase64(byteCount) {
var arr = new Uint8Array(byteCount);
crypto.getRandomValues(arr);
var binary = '';
for (var i = 0; i < arr.length; i++) {
binary += String.fromCharCode(arr[i]);
}
return btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
}
function setupCreateUserModal() {
var createUserPoliciesEl = document.getElementById('createUserPolicies');
document.querySelectorAll('[data-create-policy-template]').forEach(function(button) {
button.addEventListener('click', function() {
applyPolicyTemplate(button.dataset.createPolicyTemplate, createUserPoliciesEl);
});
});
var genAccessKeyBtn = document.getElementById('generateAccessKeyBtn');
if (genAccessKeyBtn) {
genAccessKeyBtn.addEventListener('click', function() {
var input = document.getElementById('createUserAccessKey');
if (input) input.value = generateSecureHex(8);
});
}
var genSecretKeyBtn = document.getElementById('generateSecretKeyBtn');
if (genSecretKeyBtn) {
genSecretKeyBtn.addEventListener('click', function() {
var input = document.getElementById('createUserSecretKey');
if (input) input.value = generateSecureBase64(24);
});
}
}
function setupEditUserModal() {
var editUserForm = document.getElementById('editUserForm');
var editUserDisplayName = document.getElementById('editUserDisplayName');
document.querySelectorAll('[data-edit-user]').forEach(function(btn) {
btn.addEventListener('click', function() {
var key = btn.dataset.userId;
var accessKey = btn.dataset.accessKey || key;
var name = btn.dataset.displayName;
currentEditKey = key;
currentEditAccessKey = accessKey;
editUserDisplayName.value = name;
editUserForm.action = buildUserUrl(endpoints.updateUser, key);
editUserModal.show();
});
});
}
function setupDeleteUserModal() {
var deleteUserForm = document.getElementById('deleteUserForm');
var deleteUserLabel = document.getElementById('deleteUserLabel');
var deleteSelfWarning = document.getElementById('deleteSelfWarning');
document.querySelectorAll('[data-delete-user]').forEach(function(btn) {
btn.addEventListener('click', function() {
var key = btn.dataset.userId;
var accessKey = btn.dataset.accessKey || key;
currentDeleteKey = key;
currentDeleteAccessKey = accessKey;
deleteUserLabel.textContent = accessKey;
deleteUserForm.action = buildUserUrl(endpoints.deleteUser, key);
if (accessKey === currentUserKey) {
deleteSelfWarning.classList.remove('d-none');
} else {
deleteSelfWarning.classList.add('d-none');
}
deleteUserModal.show();
});
});
}
function setupRotateSecretModal() {
var rotateUserLabel = document.getElementById('rotateUserLabel');
var confirmRotateBtn = document.getElementById('confirmRotateBtn');
var rotateCancelBtn = document.getElementById('rotateCancelBtn');
var rotateDoneBtn = document.getElementById('rotateDoneBtn');
var rotateSecretConfirm = document.getElementById('rotateSecretConfirm');
var rotateSecretResult = document.getElementById('rotateSecretResult');
var newSecretKeyInput = document.getElementById('newSecretKey');
var copyNewSecretBtn = document.getElementById('copyNewSecret');
document.querySelectorAll('[data-rotate-user]').forEach(function(btn) {
btn.addEventListener('click', function() {
currentRotateKey = btn.dataset.userId;
rotateUserLabel.textContent = btn.dataset.accessKey || currentRotateKey;
rotateSecretConfirm.classList.remove('d-none');
rotateSecretResult.classList.add('d-none');
confirmRotateBtn.classList.remove('d-none');
rotateCancelBtn.classList.remove('d-none');
rotateDoneBtn.classList.add('d-none');
rotateSecretModal.show();
});
});
if (confirmRotateBtn) {
confirmRotateBtn.addEventListener('click', async function() {
if (!currentRotateKey) return;
window.UICore.setButtonLoading(confirmRotateBtn, true, 'Rotating...');
try {
var url = buildUserUrl(endpoints.rotateSecret, currentRotateKey);
var response = await fetch(url, {
method: 'POST',
headers: {
'Accept': 'application/json',
'X-CSRFToken': csrfToken
}
});
if (!response.ok) {
var data = await response.json();
throw new Error(data.error || 'Failed to rotate secret');
}
var data = await response.json();
newSecretKeyInput.value = data.secret_key;
rotateSecretConfirm.classList.add('d-none');
rotateSecretResult.classList.remove('d-none');
confirmRotateBtn.classList.add('d-none');
rotateCancelBtn.classList.add('d-none');
rotateDoneBtn.classList.remove('d-none');
} catch (err) {
if (window.showToast) {
window.showToast(err.message, 'Error', 'danger');
}
rotateSecretModal.hide();
} finally {
window.UICore.setButtonLoading(confirmRotateBtn, false);
}
});
}
if (copyNewSecretBtn) {
copyNewSecretBtn.addEventListener('click', async function() {
await window.UICore.copyToClipboard(newSecretKeyInput.value, copyNewSecretBtn, 'Copy');
});
}
if (rotateDoneBtn) {
rotateDoneBtn.addEventListener('click', function() {
window.location.reload();
});
}
}
function openExpiryModal(key, expiresAt) {
currentExpiryKey = key;
var user = getUserByIdentifier(key);
var label = document.getElementById('expiryUserLabel');
var input = document.getElementById('expiryDateInput');
var form = document.getElementById('expiryForm');
if (label) label.textContent = currentExpiryAccessKey || (user ? user.access_key : key);
if (expiresAt) {
try {
var dt = new Date(expiresAt);
var local = new Date(dt.getTime() - dt.getTimezoneOffset() * 60000);
if (input) input.value = local.toISOString().slice(0, 16);
} catch(e) {
if (input) input.value = '';
}
} else {
if (input) input.value = '';
}
if (form) form.action = buildUserUrl(endpoints.updateExpiry, key);
var modalEl = document.getElementById('expiryModal');
if (modalEl) {
var modal = bootstrap.Modal.getOrCreateInstance(modalEl);
modal.show();
}
}
function setupExpiryModal() {
document.querySelectorAll('[data-expiry-user]').forEach(function(btn) {
btn.addEventListener('click', function(e) {
e.preventDefault();
currentExpiryAccessKey = btn.dataset.accessKey || btn.dataset.userId;
openExpiryModal(btn.dataset.userId, btn.dataset.expiresAt || '');
});
});
document.querySelectorAll('[data-expiry-preset]').forEach(function(btn) {
btn.addEventListener('click', function() {
var preset = btn.dataset.expiryPreset;
var input = document.getElementById('expiryDateInput');
if (!input) return;
if (preset === 'clear') {
input.value = '';
return;
}
var now = new Date();
var ms = 0;
if (preset === '1h') ms = 3600000;
else if (preset === '24h') ms = 86400000;
else if (preset === '7d') ms = 7 * 86400000;
else if (preset === '30d') ms = 30 * 86400000;
else if (preset === '90d') ms = 90 * 86400000;
var future = new Date(now.getTime() + ms);
var local = new Date(future.getTime() - future.getTimezoneOffset() * 60000);
input.value = local.toISOString().slice(0, 16);
});
});
var expiryForm = document.getElementById('expiryForm');
if (expiryForm) {
expiryForm.addEventListener('submit', function(e) {
e.preventDefault();
window.UICore.submitFormAjax(expiryForm, {
successMessage: 'Expiry updated',
onSuccess: function() {
var modalEl = document.getElementById('expiryModal');
if (modalEl) bootstrap.Modal.getOrCreateInstance(modalEl).hide();
window.location.reload();
}
});
});
}
}
function createUserCardHtml(user) {
var userId = user.user_id || '';
var accessKey = user.access_key || userId;
var displayName = user.display_name || accessKey;
var policies = user.policies || [];
var expiresAt = user.expires_at || '';
var admin = isAdminUser(policies);
var cardClass = 'card h-100 iam-user-card' + (admin ? ' iam-admin-card' : '');
var roleBadge = admin
? '<span class="iam-role-badge iam-role-admin" data-role-badge>Admin</span>'
: '<span class="iam-role-badge iam-role-user" data-role-badge>User</span>';
var policyBadges = '';
if (policies && policies.length > 0) {
policyBadges = policies.map(function(p) {
var bucketLabel = getBucketLabel(p.bucket);
var permLevel = getPermissionLevel(p.actions);
return '<span class="iam-perm-badge">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10" fill="currentColor" class="me-1" viewBox="0 0 16 16">' +
'<path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/>' +
'</svg>' + window.UICore.escapeHtml(bucketLabel) + ' &middot; ' + window.UICore.escapeHtml(permLevel) + '</span>';
}).join('');
} else {
policyBadges = '<span class="badge bg-secondary bg-opacity-10 text-secondary">No policies</span>';
}
var esc = window.UICore.escapeHtml;
return '<div class="col-md-6 col-xl-4 iam-user-item" data-user-id="' + esc(userId) + '" data-access-key="' + esc(accessKey) + '" data-display-name="' + esc(displayName.toLowerCase()) + '" data-access-key-filter="' + esc(accessKey.toLowerCase()) + '">' +
'<div class="' + cardClass + '">' +
'<div class="card-body">' +
'<div class="d-flex align-items-start justify-content-between mb-3">' +
'<div class="d-flex align-items-center gap-3 min-width-0 overflow-hidden">' +
'<div class="user-avatar user-avatar-lg flex-shrink-0">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" viewBox="0 0 16 16">' +
'<path d="M8 8a3 3 0 1 0 0-6 3 3 0 0 0 0 6zm2-3a2 2 0 1 1-4 0 2 2 0 0 1 4 0zm4 8c0 1-1 1-1 1H3s-1 0-1-1 1-4 6-4 6 3 6 4zm-1-.004c-.001-.246-.154-.986-.832-1.664C11.516 10.68 10.289 10 8 10c-2.29 0-3.516.68-4.168 1.332-.678.678-.83 1.418-.832 1.664h10z"/>' +
'</svg></div>' +
'<div class="min-width-0">' +
'<div class="d-flex align-items-center gap-2 mb-0">' +
'<h6 class="fw-semibold mb-0 text-truncate" title="' + esc(displayName) + '">' + esc(displayName) + '</h6>' +
roleBadge +
'</div>' +
'<div class="d-flex align-items-center gap-1">' +
'<code class="small text-muted text-truncate" title="' + esc(accessKey) + '">' + esc(accessKey) + '</code>' +
'<button type="button" class="iam-copy-key" title="Copy access key" data-copy-access-key="' + esc(accessKey) + '">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" viewBox="0 0 16 16">' +
'<path d="M4 1.5H3a2 2 0 0 0-2 2V14a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2V3.5a2 2 0 0 0-2-2h-1v1h1a1 1 0 0 1 1 1V14a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V3.5a1 1 0 0 1 1-1h1v-1z"/>' +
'<path d="M9.5 1a.5.5 0 0 1 .5.5v1a.5.5 0 0 1-.5.5h-3a.5.5 0 0 1-.5-.5v-1a.5.5 0 0 1 .5-.5h3zm-3-1A1.5 1.5 0 0 0 5 1.5v1A1.5 1.5 0 0 0 6.5 4h3A1.5 1.5 0 0 0 11 2.5v-1A1.5 1.5 0 0 0 9.5 0h-3z"/>' +
'</svg></button>' +
'</div>' +
'</div></div>' +
'<div class="dropdown flex-shrink-0">' +
'<button class="btn btn-sm btn-icon" type="button" data-bs-toggle="dropdown" aria-expanded="false">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16">' +
'<path d="M9.5 13a1.5 1.5 0 1 1-3 0 1.5 1.5 0 0 1 3 0zm0-5a1.5 1.5 0 1 1-3 0 1.5 1.5 0 0 1 3 0zm0-5a1.5 1.5 0 1 1-3 0 1.5 1.5 0 0 1 3 0z"/>' +
'</svg></button>' +
'<ul class="dropdown-menu dropdown-menu-end">' +
'<li><button class="dropdown-item" type="button" data-edit-user data-user-id="' + esc(userId) + '" data-access-key="' + esc(accessKey) + '" data-display-name="' + esc(displayName) + '">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16"><path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/></svg>Edit Name</button></li>' +
'<li><button class="dropdown-item" type="button" data-expiry-user data-user-id="' + esc(userId) + '" data-access-key="' + esc(accessKey) + '" data-expires-at="' + esc(expiresAt) + '">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16"><path d="M8 3.5a.5.5 0 0 0-1 0V9a.5.5 0 0 0 .252.434l3.5 2a.5.5 0 0 0 .496-.868L8 8.71V3.5z"/><path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm7-8A7 7 0 1 1 1 8a7 7 0 0 1 14 0z"/></svg>Set Expiry</button></li>' +
'<li><button class="dropdown-item" type="button" data-rotate-user data-user-id="' + esc(userId) + '" data-access-key="' + esc(accessKey) + '">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16"><path d="M11.534 7h3.932a.25.25 0 0 1 .192.41l-1.966 2.36a.25.25 0 0 1-.384 0l-1.966-2.36a.25.25 0 0 1 .192-.41zm-11 2h3.932a.25.25 0 0 0 .192-.41L2.692 6.23a.25.25 0 0 0-.384 0L.342 8.59A.25.25 0 0 0 .534 9z"/><path fill-rule="evenodd" d="M8 3c-1.552 0-2.94.707-3.857 1.818a.5.5 0 1 1-.771-.636A6.002 6.002 0 0 1 13.917 7H12.9A5.002 5.002 0 0 0 8 3zM3.1 9a5.002 5.002 0 0 0 8.757 2.182.5.5 0 1 1 .771.636A6.002 6.002 0 0 1 2.083 9H3.1z"/></svg>Rotate Secret</button></li>' +
'<li><hr class="dropdown-divider"></li>' +
'<li><button class="dropdown-item text-danger" type="button" data-delete-user data-user-id="' + esc(userId) + '" data-access-key="' + esc(accessKey) + '">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16"><path d="M5.5 5.5a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0v-6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0v-6a.5.5 0 0 1 .5-.5zm3 .5v6a.5.5 0 0 1-1 0v-6a.5.5 0 0 1 1 0z"/><path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/></svg>Delete User</button></li>' +
'</ul></div></div>' +
'<div class="mb-3">' +
'<div class="small text-muted mb-2">Bucket Permissions</div>' +
'<div class="d-flex flex-wrap gap-1" data-policy-badges>' + policyBadges + '</div></div>' +
'<button class="btn btn-outline-primary btn-sm w-100" type="button" data-policy-editor data-user-id="' + esc(userId) + '" data-access-key="' + esc(accessKey) + '">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16"><path d="M8 4.754a3.246 3.246 0 1 0 0 6.492 3.246 3.246 0 0 0 0-6.492zM5.754 8a2.246 2.246 0 1 1 4.492 0 2.246 2.246 0 0 1-4.492 0z"/><path d="M9.796 1.343c-.527-1.79-3.065-1.79-3.592 0l-.094.319a.873.873 0 0 1-1.255.52l-.292-.16c-1.64-.892-3.433.902-2.54 2.541l.159.292a.873.873 0 0 1-.52 1.255l-.319.094c-1.79.527-1.79 3.065 0 3.592l.319.094a.873.873 0 0 1 .52 1.255l-.16.292c-.892 1.64.901 3.434 2.541 2.54l.292-.159a.873.873 0 0 1 1.255.52l.094.319c.527 1.79 3.065 1.79 3.592 0l.094-.319a.873.873 0 0 1 1.255-.52l.292.16c1.64.893 3.434-.902 2.54-2.541l-.159-.292a.873.873 0 0 1 .52-1.255l.319-.094c1.79-.527 1.79-3.065 0-3.592l-.319-.094a.873.873 0 0 1-.52-1.255l.16-.292c.893-1.64-.902-3.433-2.541-2.54l-.292.159a.873.873 0 0 1-1.255-.52l-.094-.319z"/></svg>Manage Policies</button>' +
'</div></div></div>';
}
function attachUserCardHandlers(cardElement, user) {
var userId = user.user_id;
var accessKey = user.access_key;
var displayName = user.display_name;
var expiresAt = user.expires_at || '';
var editBtn = cardElement.querySelector('[data-edit-user]');
if (editBtn) {
editBtn.addEventListener('click', function() {
currentEditKey = userId;
currentEditAccessKey = accessKey;
document.getElementById('editUserDisplayName').value = displayName;
document.getElementById('editUserForm').action = buildUserUrl(endpoints.updateUser, userId);
editUserModal.show();
});
}
var deleteBtn = cardElement.querySelector('[data-delete-user]');
if (deleteBtn) {
deleteBtn.addEventListener('click', function() {
currentDeleteKey = userId;
currentDeleteAccessKey = accessKey;
document.getElementById('deleteUserLabel').textContent = accessKey;
document.getElementById('deleteUserForm').action = buildUserUrl(endpoints.deleteUser, userId);
var deleteSelfWarning = document.getElementById('deleteSelfWarning');
if (accessKey === currentUserKey) {
deleteSelfWarning.classList.remove('d-none');
} else {
deleteSelfWarning.classList.add('d-none');
}
deleteUserModal.show();
});
}
var rotateBtn = cardElement.querySelector('[data-rotate-user]');
if (rotateBtn) {
rotateBtn.addEventListener('click', function() {
currentRotateKey = userId;
document.getElementById('rotateUserLabel').textContent = accessKey;
document.getElementById('rotateSecretConfirm').classList.remove('d-none');
document.getElementById('rotateSecretResult').classList.add('d-none');
document.getElementById('confirmRotateBtn').classList.remove('d-none');
document.getElementById('rotateCancelBtn').classList.remove('d-none');
document.getElementById('rotateDoneBtn').classList.add('d-none');
rotateSecretModal.show();
});
}
var expiryBtn = cardElement.querySelector('[data-expiry-user]');
if (expiryBtn) {
expiryBtn.addEventListener('click', function(e) {
e.preventDefault();
currentExpiryAccessKey = accessKey;
openExpiryModal(userId, expiresAt);
});
}
var policyBtn = cardElement.querySelector('[data-policy-editor]');
if (policyBtn) {
policyBtn.addEventListener('click', function() {
document.getElementById('policyEditorUserLabel').textContent = accessKey;
document.getElementById('policyEditorUserId').value = userId;
document.getElementById('policyEditorDocument').value = getUserPolicies(userId);
policyModal.show();
});
}
var copyBtn = cardElement.querySelector('[data-copy-access-key]');
if (copyBtn) {
copyBtn.addEventListener('click', function() {
copyAccessKey(copyBtn);
});
}
}
function updateUserCount() {
var countEl = document.querySelector('.card-header .text-muted.small');
if (countEl) {
var count = document.querySelectorAll('.iam-user-card').length;
countEl.textContent = count + ' user' + (count !== 1 ? 's' : '') + ' configured';
}
}
function setupFormHandlers() {
var createUserForm = document.querySelector('#createUserModal form');
if (createUserForm) {
createUserForm.addEventListener('submit', function(e) {
e.preventDefault();
window.UICore.submitFormAjax(createUserForm, {
successMessage: 'User created',
onSuccess: function(data) {
var modal = bootstrap.Modal.getInstance(document.getElementById('createUserModal'));
if (modal) modal.hide();
createUserForm.reset();
var existingAlert = document.querySelector('.alert.alert-info.border-0.shadow-sm');
if (existingAlert) existingAlert.remove();
if (data.secret_key) {
var alertHtml = '<div class="alert alert-info border-0 shadow-sm mb-4" role="alert" id="newUserSecretAlert">' +
'<div class="d-flex align-items-start gap-2 mb-2">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="bi bi-key flex-shrink-0 mt-1" viewBox="0 0 16 16">' +
'<path d="M0 8a4 4 0 0 1 7.465-2H14a.5.5 0 0 1 .354.146l1.5 1.5a.5.5 0 0 1 0 .708l-1.5 1.5a.5.5 0 0 1-.708 0L13 9.207l-.646.647a.5.5 0 0 1-.708 0L11 9.207l-.646.647a.5.5 0 0 1-.708 0L9 9.207l-.646.647A.5.5 0 0 1 8 10h-.535A4 4 0 0 1 0 8zm4-3a3 3 0 1 0 2.712 4.285A.5.5 0 0 1 7.163 9h.63l.853-.854a.5.5 0 0 1 .708 0l.646.647.646-.647a.5.5 0 0 1 .708 0l.646.647.646-.647a.5.5 0 0 1 .708 0l.646.647.793-.793-1-1h-6.63a.5.5 0 0 1-.451-.285A3 3 0 0 0 4 5z"/><path d="M4 8a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>' +
'</svg>' +
'<div class="flex-grow-1">' +
'<div class="fw-semibold">New user created: <code>' + window.UICore.escapeHtml(data.access_key) + '</code></div>' +
'<p class="mb-2 small">These credentials are only shown once. Copy them now and store them securely.</p>' +
'</div>' +
'<button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>' +
'</div>' +
'<div class="input-group mb-2">' +
'<span class="input-group-text"><strong>Access key</strong></span>' +
'<input class="form-control font-monospace" type="text" value="' + window.UICore.escapeHtml(data.access_key) + '" readonly />' +
'<button class="btn btn-outline-primary" type="button" id="copyNewUserAccessKey">Copy</button>' +
'</div>' +
'<div class="input-group">' +
'<span class="input-group-text"><strong>Secret key</strong></span>' +
'<input class="form-control font-monospace" type="text" value="' + window.UICore.escapeHtml(data.secret_key) + '" readonly id="newUserSecret" />' +
'<button class="btn btn-outline-primary" type="button" id="copyNewUserSecret">Copy</button>' +
'</div></div>';
var container = document.querySelector('.page-header');
if (container) {
container.insertAdjacentHTML('afterend', alertHtml);
document.getElementById('copyNewUserAccessKey').addEventListener('click', async function() {
await window.UICore.copyToClipboard(data.access_key, this, 'Copy');
});
document.getElementById('copyNewUserSecret').addEventListener('click', async function() {
await window.UICore.copyToClipboard(data.secret_key, this, 'Copy');
});
}
}
var usersGrid = document.querySelector('.row.g-3');
var emptyState = document.querySelector('.empty-state');
if (emptyState) {
var emptyCol = emptyState.closest('.col-12');
if (emptyCol) emptyCol.remove();
if (!usersGrid) {
var cardBody = document.querySelector('.card-body.px-4.pb-4');
if (cardBody) {
cardBody.innerHTML = '<div class="row g-3"></div>';
usersGrid = cardBody.querySelector('.row.g-3');
}
}
}
if (usersGrid) {
var newUser = {
user_id: data.user_id,
access_key: data.access_key,
display_name: data.display_name,
expires_at: data.expires_at || '',
policies: data.policies || []
};
var cardHtml = createUserCardHtml(newUser);
usersGrid.insertAdjacentHTML('beforeend', cardHtml);
var newCard = usersGrid.lastElementChild;
attachUserCardHandlers(newCard, newUser);
users.push(newUser);
updateUserCount();
}
}
});
});
}
var policyEditorForm = document.getElementById('policyEditorForm');
if (policyEditorForm) {
policyEditorForm.addEventListener('submit', function(e) {
e.preventDefault();
var userInputEl = document.getElementById('policyEditorUserId');
var userId = userInputEl.value;
if (!userId) return;
var template = policyEditorForm.dataset.actionTemplate;
policyEditorForm.action = template.replace('USER_ID_PLACEHOLDER', encodeURIComponent(userId));
window.UICore.submitFormAjax(policyEditorForm, {
successMessage: 'Policies updated',
onSuccess: function(data) {
policyModal.hide();
var userCard = document.querySelector('.iam-user-item[data-user-id="' + userId + '"]');
if (userCard) {
var cardEl = userCard.querySelector('.iam-user-card');
var badgeContainer = cardEl ? cardEl.querySelector('[data-policy-badges]') : null;
if (badgeContainer && data.policies) {
var badges = data.policies.map(function(p) {
var bl = getBucketLabel(p.bucket);
var pl = getPermissionLevel(p.actions);
return '<span class="iam-perm-badge">' +
'<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10" fill="currentColor" class="me-1" viewBox="0 0 16 16">' +
'<path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/>' +
'</svg>' + window.UICore.escapeHtml(bl) + ' &middot; ' + window.UICore.escapeHtml(pl) + '</span>';
}).join('');
badgeContainer.innerHTML = badges || '<span class="badge bg-secondary bg-opacity-10 text-secondary">No policies</span>';
}
if (cardEl) {
var nowAdmin = isAdminUser(data.policies);
cardEl.classList.toggle('iam-admin-card', nowAdmin);
var roleBadgeEl = cardEl.querySelector('[data-role-badge]');
if (roleBadgeEl) {
if (nowAdmin) {
roleBadgeEl.className = 'iam-role-badge iam-role-admin';
roleBadgeEl.textContent = 'Admin';
} else {
roleBadgeEl.className = 'iam-role-badge iam-role-user';
roleBadgeEl.textContent = 'User';
}
}
}
}
var userIndex = users.findIndex(function(u) { return u.user_id === userId; });
if (userIndex >= 0 && data.policies) {
users[userIndex].policies = data.policies;
}
}
});
});
}
var editUserForm = document.getElementById('editUserForm');
if (editUserForm) {
editUserForm.addEventListener('submit', function(e) {
e.preventDefault();
var key = currentEditKey;
window.UICore.submitFormAjax(editUserForm, {
successMessage: 'User updated',
onSuccess: function(data) {
editUserModal.hide();
var newName = data.display_name || document.getElementById('editUserDisplayName').value;
var editBtn = document.querySelector('[data-edit-user][data-user-id="' + key + '"]');
if (editBtn) {
editBtn.setAttribute('data-display-name', newName);
var card = editBtn.closest('.iam-user-card');
if (card) {
var nameEl = card.querySelector('h6');
if (nameEl) {
nameEl.textContent = newName;
nameEl.title = newName;
}
var itemWrapper = card.closest('.iam-user-item');
if (itemWrapper) {
itemWrapper.setAttribute('data-display-name', newName.toLowerCase());
}
}
}
var userIndex = users.findIndex(function(u) { return u.user_id === key; });
if (userIndex >= 0) {
users[userIndex].display_name = newName;
}
if (currentEditAccessKey === currentUserKey) {
document.querySelectorAll('.sidebar-user .user-name').forEach(function(el) {
var truncated = newName.length > 16 ? newName.substring(0, 16) + '...' : newName;
el.textContent = truncated;
el.title = newName;
});
document.querySelectorAll('.sidebar-user[data-username]').forEach(function(el) {
el.setAttribute('data-username', newName);
});
}
}
});
});
}
var deleteUserForm = document.getElementById('deleteUserForm');
if (deleteUserForm) {
deleteUserForm.addEventListener('submit', function(e) {
e.preventDefault();
var key = currentDeleteKey;
window.UICore.submitFormAjax(deleteUserForm, {
successMessage: 'User deleted',
onSuccess: function(data) {
deleteUserModal.hide();
if (currentDeleteAccessKey === currentUserKey) {
window.location.href = '/ui/';
return;
}
var deleteBtn = document.querySelector('[data-delete-user][data-user-id="' + key + '"]');
if (deleteBtn) {
var cardCol = deleteBtn.closest('[class*="col-"]');
if (cardCol) {
cardCol.remove();
}
}
users = users.filter(function(u) { return u.user_id !== key; });
updateUserCount();
}
});
});
}
}
function setupSearch() {
var searchInput = document.getElementById('iam-user-search');
if (!searchInput) return;
searchInput.addEventListener('input', function() {
var query = searchInput.value.toLowerCase().trim();
var items = document.querySelectorAll('.iam-user-item');
var noResults = document.getElementById('iam-no-results');
var visibleCount = 0;
items.forEach(function(item) {
var name = item.getAttribute('data-display-name') || '';
var key = item.getAttribute('data-access-key-filter') || '';
var matches = !query || name.indexOf(query) >= 0 || key.indexOf(query) >= 0;
item.classList.toggle('d-none', !matches);
if (matches) visibleCount++;
});
if (noResults) {
noResults.classList.toggle('d-none', visibleCount > 0);
}
});
}
function copyAccessKey(btn) {
var key = btn.getAttribute('data-copy-access-key');
if (!key) return;
var originalHtml = btn.innerHTML;
navigator.clipboard.writeText(key).then(function() {
btn.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" viewBox="0 0 16 16"><path d="M13.854 3.646a.5.5 0 0 1 0 .708l-7 7a.5.5 0 0 1-.708 0l-3.5-3.5a.5.5 0 1 1 .708-.708L6.5 10.293l6.646-6.647a.5.5 0 0 1 .708 0z"/></svg>';
btn.style.color = '#22c55e';
setTimeout(function() {
btn.innerHTML = originalHtml;
btn.style.color = '';
}, 1200);
}).catch(function() {});
}
function setupCopyAccessKeyButtons() {
document.querySelectorAll('[data-copy-access-key]').forEach(function(btn) {
btn.addEventListener('click', function() {
copyAccessKey(btn);
});
});
}
return {
init: init
};
})();

334
rust/myfsio-engine/crates/myfsio-server/static/js/ui-core.js Normal file
View File

@@ -0,0 +1,334 @@
window.UICore = (function() {
'use strict';
function getCsrfToken() {
const meta = document.querySelector('meta[name="csrf-token"]');
return meta ? meta.getAttribute('content') : '';
}
function formatBytes(bytes) {
if (!Number.isFinite(bytes)) return bytes + ' bytes';
const units = ['bytes', 'KB', 'MB', 'GB', 'TB'];
let i = 0;
let size = bytes;
while (size >= 1024 && i < units.length - 1) {
size /= 1024;
i++;
}
return size.toFixed(i === 0 ? 0 : 1) + ' ' + units[i];
}
function escapeHtml(value) {
if (value === null || value === undefined) return '';
return String(value)
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#039;');
}
async function submitFormAjax(form, options) {
options = options || {};
var onSuccess = options.onSuccess || function() {};
var onError = options.onError || function() {};
var successMessage = options.successMessage || 'Operation completed';
var formData = new FormData(form);
var hasFileInput = !!form.querySelector('input[type="file"]');
var requestBody = hasFileInput ? formData : new URLSearchParams(formData);
var csrfToken = getCsrfToken();
var submitBtn = form.querySelector('[type="submit"]');
var originalHtml = submitBtn ? submitBtn.innerHTML : '';
try {
if (submitBtn) {
submitBtn.disabled = true;
submitBtn.innerHTML = '<span class="spinner-border spinner-border-sm me-1"></span>Saving...';
}
var formAction = form.getAttribute('action') || form.action;
var headers = {
'X-CSRF-Token': csrfToken,
'Accept': 'application/json',
'X-Requested-With': 'XMLHttpRequest'
};
if (!hasFileInput) {
headers['Content-Type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
}
var response = await fetch(formAction, {
method: form.getAttribute('method') || 'POST',
headers: headers,
body: requestBody,
redirect: 'follow'
});
var contentType = response.headers.get('content-type') || '';
if (!contentType.includes('application/json')) {
throw new Error('Server returned an unexpected response. Please try again.');
}
var data = await response.json();
if (!response.ok) {
throw new Error(data.error || 'HTTP ' + response.status);
}
window.showToast(data.message || successMessage, 'Success', 'success');
onSuccess(data);
} catch (err) {
window.showToast(err.message, 'Error', 'error');
onError(err);
} finally {
if (submitBtn) {
submitBtn.disabled = false;
submitBtn.innerHTML = originalHtml;
}
}
}
function PollingManager() {
this.intervals = {};
this.callbacks = {};
this.timers = {};
this.defaults = {
replication: 30000,
lifecycle: 60000,
connectionHealth: 60000,
bucketStats: 120000
};
this._loadSettings();
}
PollingManager.prototype._loadSettings = function() {
try {
var stored = localStorage.getItem('myfsio-polling-intervals');
if (stored) {
var settings = JSON.parse(stored);
for (var key in settings) {
if (settings.hasOwnProperty(key)) {
this.defaults[key] = settings[key];
}
}
}
} catch (e) {
console.warn('Failed to load polling settings:', e);
}
};
PollingManager.prototype.saveSettings = function(settings) {
try {
for (var key in settings) {
if (settings.hasOwnProperty(key)) {
this.defaults[key] = settings[key];
}
}
localStorage.setItem('myfsio-polling-intervals', JSON.stringify(this.defaults));
} catch (e) {
console.warn('Failed to save polling settings:', e);
}
};
PollingManager.prototype.start = function(key, callback, interval) {
this.stop(key);
var ms = interval !== undefined ? interval : (this.defaults[key] || 30000);
if (ms <= 0) return;
this.callbacks[key] = callback;
this.intervals[key] = ms;
callback();
var self = this;
this.timers[key] = setInterval(function() {
if (!document.hidden) {
callback();
}
}, ms);
};
PollingManager.prototype.stop = function(key) {
if (this.timers[key]) {
clearInterval(this.timers[key]);
delete this.timers[key];
}
};
PollingManager.prototype.stopAll = function() {
for (var key in this.timers) {
if (this.timers.hasOwnProperty(key)) {
clearInterval(this.timers[key]);
}
}
this.timers = {};
};
PollingManager.prototype.updateInterval = function(key, newInterval) {
var callback = this.callbacks[key];
this.defaults[key] = newInterval;
this.saveSettings(this.defaults);
if (callback) {
this.start(key, callback, newInterval);
}
};
PollingManager.prototype.getSettings = function() {
var result = {};
for (var key in this.defaults) {
if (this.defaults.hasOwnProperty(key)) {
result[key] = this.defaults[key];
}
}
return result;
};
var pollingManager = new PollingManager();
document.addEventListener('visibilitychange', function() {
if (document.hidden) {
pollingManager.stopAll();
} else {
for (var key in pollingManager.callbacks) {
if (pollingManager.callbacks.hasOwnProperty(key)) {
pollingManager.start(key, pollingManager.callbacks[key], pollingManager.intervals[key]);
}
}
}
});
window.addEventListener('beforeunload', function() {
pollingManager.stopAll();
});
return {
getCsrfToken: getCsrfToken,
formatBytes: formatBytes,
escapeHtml: escapeHtml,
submitFormAjax: submitFormAjax,
PollingManager: PollingManager,
pollingManager: pollingManager
};
})();
window.pollingManager = window.UICore.pollingManager;
window.UICore.copyToClipboard = async function(text, button, originalText) {
try {
await navigator.clipboard.writeText(text);
if (button) {
var prevText = button.textContent;
button.textContent = 'Copied!';
setTimeout(function() {
button.textContent = originalText || prevText;
}, 1500);
}
return true;
} catch (err) {
console.error('Copy failed:', err);
return false;
}
};
window.UICore.setButtonLoading = function(button, isLoading, loadingText) {
if (!button) return;
if (isLoading) {
button._originalHtml = button.innerHTML;
button._originalDisabled = button.disabled;
button.disabled = true;
button.innerHTML = '<span class="spinner-border spinner-border-sm me-1"></span>' + (loadingText || 'Loading...');
} else {
button.disabled = button._originalDisabled || false;
button.innerHTML = button._originalHtml || button.innerHTML;
}
};
window.UICore.updateBadgeCount = function(selector, count, singular, plural) {
var badge = document.querySelector(selector);
if (badge) {
var label = count === 1 ? (singular || '') : (plural || 's');
badge.textContent = count + ' ' + label;
}
};
window.UICore.setupJsonAutoIndent = function(textarea) {
if (!textarea) return;
textarea.addEventListener('keydown', function(e) {
if (e.key === 'Enter') {
e.preventDefault();
var start = this.selectionStart;
var end = this.selectionEnd;
var value = this.value;
var lineStart = value.lastIndexOf('\n', start - 1) + 1;
var currentLine = value.substring(lineStart, start);
var indentMatch = currentLine.match(/^(\s*)/);
var indent = indentMatch ? indentMatch[1] : '';
var trimmedLine = currentLine.trim();
var lastChar = trimmedLine.slice(-1);
var newIndent = indent;
var insertAfter = '';
if (lastChar === '{' || lastChar === '[') {
newIndent = indent + ' ';
var charAfterCursor = value.substring(start, start + 1).trim();
if ((lastChar === '{' && charAfterCursor === '}') ||
(lastChar === '[' && charAfterCursor === ']')) {
insertAfter = '\n' + indent;
}
} else if (lastChar === ',' || lastChar === ':') {
newIndent = indent;
}
var insertion = '\n' + newIndent + insertAfter;
var newValue = value.substring(0, start) + insertion + value.substring(end);
this.value = newValue;
var newCursorPos = start + 1 + newIndent.length;
this.selectionStart = this.selectionEnd = newCursorPos;
this.dispatchEvent(new Event('input', { bubbles: true }));
}
if (e.key === 'Tab') {
e.preventDefault();
var start = this.selectionStart;
var end = this.selectionEnd;
if (e.shiftKey) {
var lineStart = this.value.lastIndexOf('\n', start - 1) + 1;
var lineContent = this.value.substring(lineStart, start);
if (lineContent.startsWith(' ')) {
this.value = this.value.substring(0, lineStart) +
this.value.substring(lineStart + 2);
this.selectionStart = this.selectionEnd = Math.max(lineStart, start - 2);
}
} else {
this.value = this.value.substring(0, start) + ' ' + this.value.substring(end);
this.selectionStart = this.selectionEnd = start + 2;
}
this.dispatchEvent(new Event('input', { bubbles: true }));
}
});
};
document.addEventListener('DOMContentLoaded', function() {
var flashMessage = sessionStorage.getItem('flashMessage');
if (flashMessage) {
sessionStorage.removeItem('flashMessage');
try {
var msg = JSON.parse(flashMessage);
if (window.showToast) {
window.showToast(msg.body || msg.title, msg.title, msg.variant || 'info');
}
} catch (e) {}
}
});

6
rust/myfsio-engine/crates/myfsio-server/templates/base.html
View File

@@ -3,7 +3,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
{% if principal %}<meta name="csrf-token" content="{{ csrf_token() }}" />{% endif %}
{% if principal %}<meta name="csrf-token" content="{{ csrf_token_value }}" />{% endif %}
<title>MyFSIO Console</title>
<link rel="icon" type="image/png" href="{{ url_for(endpoint="static", filename="images/MyFSIO.png") }}" />
<link rel="icon" type="image/x-icon" href="{{ url_for(endpoint="static", filename="images/MyFSIO.ico") }}" />
@@ -145,7 +145,7 @@
</div>
</div>
<form method="post" action="{{ url_for(endpoint="ui.logout") }}" class="w-100">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<button class="sidebar-logout-btn" type="submit">
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" fill="currentColor" viewBox="0 0 16 16">
<path fill-rule="evenodd" d="M10 12.5a.5.5 0 0 1-.5.5h-8a.5.5 0 0 1-.5-.5v-9a.5.5 0 0 1 .5-.5h8a.5.5 0 0 1 .5.5v2a.5.5 0 0 0 1 0v-2A1.5 1.5 0 0 0 9.5 2h-8A1.5 1.5 0 0 0 0 3.5v9A1.5 1.5 0 0 0 1.5 14h8a1.5 1.5 0 0 0 1.5-1.5v-2a.5.5 0 0 0-1 0v2z"/>
@@ -264,7 +264,7 @@
</div>
</div>
<form method="post" action="{{ url_for(endpoint="ui.logout") }}" class="w-100">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<button class="sidebar-logout-btn" type="submit">
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" fill="currentColor" viewBox="0 0 16 16">
<path fill-rule="evenodd" d="M10 12.5a.5.5 0 0 1-.5.5h-8a.5.5 0 0 1-.5-.5v-9a.5.5 0 0 1 .5-.5h8a.5.5 0 0 1 .5.5v2a.5.5 0 0 0 1 0v-2A1.5 1.5 0 0 0 9.5 2h-8A1.5 1.5 0 0 0 0 3.5v9A1.5 1.5 0 0 0 1.5 14h8a1.5 1.5 0 0 0 1.5-1.5v-2a.5.5 0 0 0-1 0v2z"/>

108
rust/myfsio-engine/crates/myfsio-server/templates/bucket_detail.html
View File

@@ -395,14 +395,8 @@
{% endif %}
{% if can_edit_policy %}
{% set preset_choice = "custom" %}
{% if not bucket_policy %}
{% set preset_choice = "private" %}
{% elif bucket_policy_text and bucket_policy_text | trim == default_policy | trim %}
{% set preset_choice = "public" %}
{% endif %}
<form method="post" action="{{ url_for(endpoint="ui.update_bucket_policy", bucket_name=bucket_name) }}" id="bucketPolicyForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<input type="hidden" name="mode" value="upsert" id="policyMode" />
<div class="mb-4">
@@ -454,7 +448,7 @@
</button>
</label>
<div class="position-relative">
<textarea class="form-control font-monospace" rows="14" name="policy_document" id="policyDocument" data-public-template='{{ default_policy | json_encode | safe }}' spellcheck="false" style="font-size: 0.85rem; line-height: 1.5; tab-size: 2;">{{ bucket_policy_text or default_policy }}</textarea>
<textarea class="form-control font-monospace" rows="14" name="policy_document" id="policyDocument" data-public-template='{{ default_policy | json_encode | safe }}' spellcheck="false" style="font-size: 0.85rem; line-height: 1.5; tab-size: 2;">{% if bucket_policy_text %}{{ bucket_policy_text }}{% else %}{{ default_policy }}{% endif %}</textarea>
<div id="policyValidationStatus" class="position-absolute top-0 end-0 m-2 d-none">
<span class="badge bg-success-subtle text-success" id="policyValidBadge">
<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" class="me-1" viewBox="0 0 16 16">
@@ -655,7 +649,7 @@
</button>
{% else %}
<form method="post" action="{{ url_for(endpoint="ui.update_bucket_versioning", bucket_name=bucket_name) }}" id="enableVersioningForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<input type="hidden" name="state" value="enable" />
<button class="btn btn-success" type="submit">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
@@ -783,7 +777,7 @@
{% if can_manage_encryption %}
<form method="post" action="{{ url_for(endpoint="ui.update_bucket_encryption", bucket_name=bucket_name) }}" id="encryptionForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<input type="hidden" name="action" value="enable" id="encryptionAction" />
<div class="mb-3">
@@ -817,7 +811,7 @@
<option value="">Use default KMS key</option>
{% for key in kms_keys %}
<option value="{{ key.key_id }}" {% if key.key_id == enc_kms_key %}selected{% else %}{% endif %}>
{{ key.description or key.key_id }} ({{ key.key_id | slice(start=0, end=8) }}...)
{% if key.description %}{{ key.description }}{% else %}{{ key.key_id }}{% endif %} ({{ key.key_id | slice(start=0, end=8) }}...)
</option>
{% endfor %}
</select>
@@ -832,14 +826,14 @@
</svg>
Save Encryption Settings
</button>
{% if enc_algorithm %}
<button type="button" class="btn btn-outline-danger" id="disableEncryptionBtn">
<button type="button" class="btn btn-outline-danger" id="disableEncryptionBtn"
data-bs-toggle="modal" data-bs-target="#disableEncryptionModal"
{% if not enc_algorithm %}style="display: none;"{% endif %}>
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
<path d="M11 1a2 2 0 0 0-2 2v4a2 2 0 0 1 2 2v5a2 2 0 0 1-2 2H3a2 2 0 0 1-2-2V9a2 2 0 0 1 2-2h5V3a3 3 0 0 1 6 0v4a.5.5 0 0 1-1 0V3a2 2 0 0 0-2-2z"/>
</svg>
Disable Encryption
</button>
{% endif %}
</div>
</form>
{% else %}
@@ -864,7 +858,6 @@
<div class="card-body">
{% set max_bytes = bucket_quota.max_bytes %}
{% set max_objects = bucket_quota.max_objects %}
{% set has_quota = max_bytes != null or max_objects != null %}
{% set current_objects = bucket_stats.objects | default(value=0) %}
{% set version_count = bucket_stats.version_count | default(value=0) %}
{% set total_objects = bucket_stats.total_objects | default(value=current_objects) %}
@@ -879,10 +872,10 @@
<div class="border rounded p-3 text-center">
<div class="fs-4 fw-bold text-primary">{{ total_objects }}</div>
<div class="small text-muted">Total Objects</div>
{% if max_objects != null %}
{% if has_max_objects %}
<div class="progress mt-2" style="height: 4px;">
{% if max_objects > 0 %}{% set obj_pct = total_objects / max_objects * 100 | int %}{% else %}{% set obj_pct = 0 %}{% endif %}
<div class="progress-bar {% if obj_pct >= 90 %}bg-danger{% elif obj_pct >= 75 %}bg-warning{% else %}bg-success{% endif %}" style="width: {{ [obj_pct, 100] | min }}%"></div>
<div class="progress-bar {% if obj_pct >= 90 %}bg-danger{% elif obj_pct >= 75 %}bg-warning{% else %}bg-success{% endif %}" style="width: {% if obj_pct > 100 %}100{% else %}{{ obj_pct }}{% endif %}%"></div>
</div>
<div class="small text-muted mt-1">{{ obj_pct }}% of {{ max_objects }} limit</div>
{% else %}
@@ -899,10 +892,10 @@
<div class="border rounded p-3 text-center">
<div class="fs-4 fw-bold text-primary">{{ total_bytes | filesizeformat }}</div>
<div class="small text-muted">Total Storage</div>
{% if max_bytes != null %}
{% if has_max_bytes %}
<div class="progress mt-2" style="height: 4px;">
{% if max_bytes > 0 %}{% set bytes_pct = total_bytes / max_bytes * 100 | int %}{% else %}{% set bytes_pct = 0 %}{% endif %}
<div class="progress-bar {% if bytes_pct >= 90 %}bg-danger{% elif bytes_pct >= 75 %}bg-warning{% else %}bg-success{% endif %}" style="width: {{ [bytes_pct, 100] | min }}%"></div>
<div class="progress-bar {% if bytes_pct >= 90 %}bg-danger{% elif bytes_pct >= 75 %}bg-warning{% else %}bg-success{% endif %}" style="width: {% if bytes_pct > 100 %}100{% else %}{{ bytes_pct }}{% endif %}%"></div>
</div>
<div class="small text-muted mt-1">{{ bytes_pct }}% of {{ max_bytes | filesizeformat }} limit</div>
{% else %}
@@ -924,14 +917,14 @@
<path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm.93-9.412-1 4.705c-.07.34.029.533.304.533.194 0 .487-.07.686-.246l-.088.416c-.287.346-.92.598-1.465.598-.703 0-1.002-.422-.808-1.319l.738-3.468c.064-.293.006-.399-.287-.47l-.451-.081.082-.381 2.29-.287zM8 5.5a1 1 0 1 1 0-2 1 1 0 0 1 0 2z"/>
</svg>
<div>
<strong>Storage quota enabled</strong>
<strong>Storage quota active</strong>
<p class="mb-0 small">
{% if max_bytes != null and max_objects != null %}
Limited to {{ max_bytes | filesizeformat }} and {{ max_objects }} objects.
{% elif max_bytes != null %}
Limited to {{ max_bytes | filesizeformat }} storage.
{% else %}
Limited to {{ max_objects }} objects.
{% if has_max_bytes and has_max_objects %}
This bucket is limited to {{ max_bytes | filesizeformat }} storage and {{ max_objects }} objects.
{% elif has_max_bytes %}
This bucket is limited to {{ max_bytes | filesizeformat }} storage.
{% elif has_max_objects %}
This bucket is limited to {{ max_objects }} objects.
{% endif %}
</p>
</div>
@@ -951,14 +944,14 @@
{% if can_manage_quota %}
<form method="post" action="{{ url_for(endpoint="ui.update_bucket_quota", bucket_name=bucket_name) }}" id="quotaForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<input type="hidden" name="action" value="set" id="quotaAction" />
<div class="mb-3">
<label for="max_mb" class="form-label fw-medium">Maximum Storage Size</label>
<div class="input-group">
<input type="number" class="form-control" id="max_mb" name="max_mb"
value="{% if max_bytes != null %}{{ (max_bytes / 1048576) | int }}{% else %}{{ "" }}{% endif %}"
value="{% if has_max_bytes %}{{ (max_bytes / 1048576) | int }}{% endif %}"
min="1" step="1" placeholder="Unlimited">
<span class="input-group-text">MB</span>
</div>
@@ -968,7 +961,7 @@
<div class="mb-4">
<label for="max_objects" class="form-label fw-medium">Maximum Object Count</label>
<input type="number" class="form-control" id="max_objects" name="max_objects"
value="{% if max_objects != null %}{{ max_objects }}{% else %}{{ "" }}{% endif %}"
value="{% if has_max_objects %}{{ max_objects }}{% endif %}"
min="0" step="1" placeholder="Unlimited">
<div class="form-text">Maximum number of objects allowed. Leave empty for unlimited.</div>
</div>
@@ -1058,7 +1051,7 @@
{% if can_manage_website %}
<form method="post" action="{{ url_for(endpoint="ui.update_bucket_website", bucket_name=bucket_name) }}" id="websiteForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<input type="hidden" name="action" value="enable" id="websiteAction" />
<div class="mb-3">
@@ -1423,7 +1416,7 @@
Refresh
</a>
<form id="pause-replication-form" method="POST" action="{{ url_for(endpoint="ui.update_bucket_replication", bucket_name=bucket_name) }}" class="d-inline">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<input type="hidden" name="action" value="pause">
<button type="submit" class="btn btn-outline-warning">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
@@ -1499,7 +1492,7 @@
<div class="d-flex gap-2">
<form method="POST" action="{{ url_for(endpoint="ui.update_bucket_replication", bucket_name=bucket_name) }}" class="d-inline">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<input type="hidden" name="action" value="resume">
<button type="submit" class="btn btn-primary">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
@@ -1538,7 +1531,7 @@
{% if is_replication_admin and connections %}
<form method="POST" action="{{ url_for(endpoint="ui.update_bucket_replication", bucket_name=bucket_name) }}">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<input type="hidden" name="action" value="create">
<div class="mb-3">
@@ -1713,7 +1706,7 @@
{% endif %}
<div class="row g-4">
<div class="col-lg-8">
<div class="card shadow-sm" id="lifecycle-rules-card" data-lifecycle-url="{{ lifecycle_url }}" data-lifecycle-enabled="{{ lifecycle_enabled|lower }}">
<div class="card shadow-sm" id="lifecycle-rules-card" data-lifecycle-url="{{ lifecycle_url }}" data-lifecycle-enabled="{% if lifecycle_enabled %}true{% else %}false{% endif %}">
<div class="card-header d-flex justify-content-between align-items-center">
<div class="d-flex align-items-center">
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" fill="currentColor" class="text-primary me-2" viewBox="0 0 16 16">
@@ -2024,7 +2017,7 @@
<div class="modal-footer border-0 pt-0">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
<form method="post" action="{{ url_for(endpoint="ui.update_bucket_policy", bucket_name=bucket_name) }}" class="d-inline" id="deletePolicyForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<input type="hidden" name="mode" value="delete" />
<button type="submit" class="btn btn-danger">Delete Policy</button>
</form>
@@ -2056,7 +2049,7 @@
data-multipart-complete-template="{{ url_for(endpoint="ui.complete_multipart_upload", bucket_name=bucket_name, upload_id="UPLOAD_ID_PLACEHOLDER") }}"
data-multipart-abort-template="{{ url_for(endpoint="ui.abort_multipart_upload", bucket_name=bucket_name, upload_id="UPLOAD_ID_PLACEHOLDER") }}"
>
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<div class="modal-body">
<p class="text-muted small mb-3">Upload files to <code>{{ bucket_name }}</code>. You can select multiple files at once.</p>
<div class="row g-3">
@@ -2212,7 +2205,7 @@
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<form method="post" action="{{ url_for(endpoint="ui.delete_bucket", bucket_name=bucket_name) }}" id="deleteBucketForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<div class="modal-body">
<div class="alert alert-danger d-flex align-items-center mb-3" role="alert">
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" fill="currentColor" class="flex-shrink-0 me-2" viewBox="0 0 16 16">
@@ -2322,7 +2315,7 @@
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<form method="post" id="deleteObjectForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<div class="modal-body">
<p class="mb-3">
Are you sure you want to delete this object?
@@ -2440,7 +2433,7 @@
<div class="modal-footer">
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
<form method="POST" action="{{ url_for(endpoint="ui.update_bucket_versioning", bucket_name=bucket_name) }}" class="d-inline" id="suspendVersioningForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<input type="hidden" name="state" value="suspend" />
<button type="submit" class="btn btn-warning">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
@@ -2454,6 +2447,41 @@
</div>
</div>
<div class="modal fade" id="disableEncryptionModal" tabindex="-1" aria-hidden="true">
<div class="modal-dialog modal-dialog-centered">
<div class="modal-content">
<div class="modal-header border-0 pb-0">
<h5 class="modal-title fw-semibold">
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-warning" viewBox="0 0 16 16">
<path d="M7.938 2.016A.13.13 0 0 1 8.002 2a.13.13 0 0 1 .063.016.146.146 0 0 1 .054.057l6.857 11.667c.036.06.035.124.002.183a.163.163 0 0 1-.054.06.116.116 0 0 1-.066.017H1.146a.115.115 0 0 1-.066-.017.163.163 0 0 1-.054-.06.176.176 0 0 1 .002-.183L7.884 2.073a.147.147 0 0 1 .054-.057zm1.044-.45a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566z"/>
<path d="M7.002 12a1 1 0 1 1 2 0 1 1 0 0 1-2 0zM7.1 5.995a.905.905 0 1 1 1.8 0l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995z"/>
</svg>
Disable Default Encryption
</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
<div class="alert alert-warning d-flex align-items-start mb-3" role="alert">
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" fill="currentColor" class="flex-shrink-0 me-2 mt-1" viewBox="0 0 16 16">
<path d="M7.938 2.016A.13.13 0 0 1 8.002 2a.13.13 0 0 1 .063.016.146.146 0 0 1 .054.057l6.857 11.667c.036.06.035.124.002.183a.163.163 0 0 1-.054.06.116.116 0 0 1-.066.017H1.146a.115.115 0 0 1-.066-.017.163.163 0 0 1-.054-.06.176.176 0 0 1 .002-.183L7.884 2.073a.147.147 0 0 1 .054-.057zm1.044-.45a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566z"/>
</svg>
<div>
<strong>Are you sure?</strong><br>
<span class="small">New objects will not be encrypted automatically.</span>
</div>
</div>
<p class="text-muted small mb-0">Existing encrypted objects remain encrypted and readable.</p>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
<button type="button" class="btn btn-danger" id="confirmDisableEncryptionBtn">
Disable Encryption
</button>
</div>
</div>
</div>
</div>
<div class="modal fade" id="disableReplicationModal" tabindex="-1" aria-hidden="true">
<div class="modal-dialog modal-dialog-centered">
<div class="modal-content">
@@ -2483,7 +2511,7 @@
<div class="modal-footer">
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
<form method="POST" action="{{ url_for(endpoint="ui.update_bucket_replication", bucket_name=bucket_name) }}" class="d-inline">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<input type="hidden" name="action" value="delete">
<button type="submit" class="btn btn-warning">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">

4
rust/myfsio-engine/crates/myfsio-server/templates/buckets.html
View File

@@ -113,10 +113,10 @@
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<form method="post" action="{{ url_for(endpoint="ui.create_bucket") }}" id="createBucketForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<div class="modal-body pt-0">
<label class="form-label fw-medium">Bucket name</label>
<input class="form-control" type="text" name="bucket_name" pattern="[a-z0-9.-]{3,63}" placeholder="my-bucket-name" required autofocus />
<input class="form-control" type="text" name="bucket_name" pattern="[a-z0-9.\-]{3,63}" placeholder="my-bucket-name" required autofocus />
<div class="form-text">Use 3-63 characters: lowercase letters, numbers, dots, or hyphens.</div>
</div>
<div class="modal-footer">

8
rust/myfsio-engine/crates/myfsio-server/templates/connections.html
View File

@@ -36,7 +36,7 @@
</div>
<div class="card-body px-4 pb-4">
<form method="POST" action="{{ url_for(endpoint="ui.create_connection") }}" id="createConnectionForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}"/>
<div class="mb-3">
<label for="name" class="form-label fw-medium">Name</label>
<input type="text" class="form-control" id="name" name="name" required placeholder="Production Backup">
@@ -197,7 +197,7 @@
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<form method="POST" id="editConnectionForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}"/>
<div class="modal-body">
<div class="mb-3">
<label for="edit_name" class="form-label fw-medium">Name</label>
@@ -274,7 +274,7 @@
<div class="modal-footer">
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
<form method="POST" id="deleteConnectionForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}"/>
<button type="submit" class="btn btn-danger">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
@@ -291,7 +291,7 @@
<script src="{{ url_for(endpoint="static", filename="js/connections-management.js") }}"></script>
<script>
ConnectionsManagement.init({
csrfToken: "{{ csrf_token() }}",
csrfToken: "{{ csrf_token_value }}",
endpoints: {
test: "{{ url_for(endpoint="ui.test_connection") }}",
updateTemplate: "{{ url_for(endpoint="ui.update_connection", connection_id="CONNECTION_ID") }}",

91
rust/myfsio-engine/crates/myfsio-server/templates/docs.html
View File

@@ -71,28 +71,35 @@
<span class="docs-section-kicker">01</span>
<h2 class="h4 mb-0">Set up &amp; run locally</h2>
</div>
<p class="text-muted">Prepare a virtual environment, install dependencies, and launch both servers for a complete console + API experience.</p>
<p class="text-muted">Build or run the Rust server and launch the API plus web UI from a single process.</p>
<div class="alert alert-light border small mb-3">
Runtime note: MyFSIO now runs from the Rust server in <code>rust/myfsio-engine</code>. For the verified runtime configuration list, use the repository <code>docs.md</code>.
</div>
<ol class="docs-steps">
<li>Install Python 3.11+ plus system build tools.</li>
<li>Create a virtual environment and install <code>requirements.txt</code>.</li>
<li>Start the services with <code>python run.py</code>.</li>
<li>Install a current Rust toolchain.</li>
<li>Change into <code>rust/myfsio-engine</code>.</li>
<li>Start the server with <code>cargo run -p myfsio-server --</code>.</li>
</ol>
<pre class="mb-3"><code class="language-bash">python -m venv .venv
. .venv/Scripts/activate # PowerShell: .\\.venv\\Scripts\\Activate.ps1
pip install -r requirements.txt
<pre class="mb-3"><code class="language-bash">cd rust/myfsio-engine
# Run both API and UI (Development)
python run.py
# Run API + UI
cargo run -p myfsio-server --
# Run in Production (Granian server)
python run.py --prod
# Show resolved configuration
cargo run -p myfsio-server -- --show-config
# Or run individually
python run.py --mode api
python run.py --mode ui
# Validate configuration
cargo run -p myfsio-server -- --check-config
# API only
UI_ENABLED=false cargo run -p myfsio-server --
# Release build
cargo build --release -p myfsio-server
./target/release/myfsio-server
</code></pre>
<h3 class="h6 mt-4 mb-2">Configuration</h3>
<p class="text-muted small">Configuration defaults live in <code>app/config.py</code>. You can override them using environment variables. This is critical for production deployments behind proxies.</p>
<p class="text-muted small">Configuration is driven by the Rust server environment. See <code>docs.md</code> for the authoritative runtime variables.</p>
<div class="table-responsive">
<table class="table table-sm table-bordered small mb-0">
<thead class="table-light">
@@ -121,15 +128,15 @@ python run.py --mode ui
<tr>
<td><code>SECRET_KEY</code></td>
<td>(Auto-generated)</td>
<td>Flask session key. Auto-generates if not set. <strong>Set explicitly in production.</strong></td>
<td>Session signing and IAM-at-rest encryption key. <strong>Set explicitly in production.</strong></td>
</tr>
<tr>
<td><code>APP_HOST</code></td>
<td><code>0.0.0.0</code></td>
<td><code>HOST</code></td>
<td><code>127.0.0.1</code></td>
<td>Bind interface.</td>
</tr>
<tr>
<td><code>APP_PORT</code></td>
<td><code>PORT</code></td>
<td><code>5000</code></td>
<td>Listen port (UI uses 5100).</td>
</tr>
@@ -454,19 +461,19 @@ python run.py --mode ui
<h3 class="h6 text-uppercase text-muted mt-4">Quick Start (nohup)</h3>
<p class="text-muted small">Simplest way to run in background—survives terminal close:</p>
<pre class="mb-3"><code class="language-bash"># Using Python
nohup python run.py --prod > /dev/null 2>&1 &
<pre class="mb-3"><code class="language-bash"># From the repository
nohup cargo run -p myfsio-server -- > /dev/null 2>&1 &
# Using compiled binary
nohup ./myfsio > /dev/null 2>&1 &
# Using a compiled binary
nohup ./myfsio-server > /dev/null 2>&1 &
# Check if running
ps aux | grep myfsio</code></pre>
ps aux | grep myfsio-server</code></pre>
<h3 class="h6 text-uppercase text-muted mt-4">Screen / Tmux</h3>
<p class="text-muted small">Attach/detach from a persistent session:</p>
<pre class="mb-3"><code class="language-bash"># Start in a detached screen session
screen -dmS myfsio ./myfsio
screen -dmS myfsio ./myfsio-server
# Attach to view logs
screen -r myfsio
@@ -507,7 +514,7 @@ sudo journalctl -u myfsio -f # View logs</code></pre>
<span class="docs-section-kicker">03</span>
<h2 class="h4 mb-0">Authenticate &amp; manage IAM</h2>
</div>
<p class="text-muted">On first startup, MyFSIO generates random admin credentials and prints them to the console. Set <code>ADMIN_ACCESS_KEY</code> and <code>ADMIN_SECRET_KEY</code> env vars for custom credentials. When <code>SECRET_KEY</code> is configured, the IAM config is encrypted at rest. To reset credentials, run <code>python run.py --reset-cred</code>.</p>
<p class="text-muted">On first startup, MyFSIO generates random admin credentials and prints them to the console. Set <code>ADMIN_ACCESS_KEY</code> and <code>ADMIN_SECRET_KEY</code> for custom credentials. When <code>SECRET_KEY</code> is configured, the IAM config is encrypted at rest. To reset credentials, run <code>cargo run -p myfsio-server -- --reset-cred</code> or the installed binary with <code>--reset-cred</code>.</p>
<div class="docs-highlight mb-3">
<ol class="mb-0">
<li>Check the console output for the generated <code>Access Key</code> and <code>Secret Key</code>, then visit <code>/ui/login</code>.</li>
@@ -517,7 +524,7 @@ sudo journalctl -u myfsio -f # View logs</code></pre>
<li>Bucket policies layer on top of IAM. Apply Private/Public presets or paste custom JSON; changes reload instantly.</li>
</ol>
</div>
<p class="mb-0 text-muted">All API calls require <code>X-Access-Key</code> and <code>X-Secret-Key</code> headers. The UI stores them in the Flask session after you log in.</p>
<p class="mb-0 text-muted">All API calls require <code>X-Access-Key</code> and <code>X-Secret-Key</code> headers. The UI stores them in the server-managed session after you log in.</p>
</div>
</article>
<article id="console" class="card shadow-sm docs-section">
@@ -737,7 +744,7 @@ curl -X PUT {{ api_base }}/demo/notes.txt \
</tbody>
</table>
</div>
<p class="small text-muted mt-3 mb-0">All responses include <code>X-Request-Id</code> for tracing. See the <a href="#api-matrix">Full API Reference</a> for the complete endpoint list. Logs land in <code>logs/api.log</code> and <code>logs/ui.log</code>.</p>
<p class="small text-muted mt-3 mb-0">All responses include <code>X-Request-Id</code> for tracing. See the <a href="#api-matrix">Full API Reference</a> for the complete endpoint list. When running the Rust server, logs go to stdout and whatever service manager or container runtime is supervising the process.</p>
</div>
</article>
<article id="examples" class="card shadow-sm docs-section">
@@ -1174,7 +1181,7 @@ export SITE_ID=us-west-1
export SITE_ENDPOINT=https://s3.us-west-1.example.com
export SITE_REGION=us-west-1
export SITE_PRIORITY=100
python run.py</code></pre>
cargo run -p myfsio-server --</code></pre>
<h3 class="h6 text-uppercase text-muted mt-4">Using the Sites UI</h3>
<p class="small text-muted">Navigate to <a href="{{ url_for(endpoint="ui.sites_dashboard") }}">Sites</a> in the sidebar to manage site configuration:</p>
@@ -1442,12 +1449,12 @@ curl -X PUT "{{ api_base }}/bucket/&lt;bucket&gt;?quota" \
<pre class="mb-2"><code class="language-bash"># PowerShell
$env:ENCRYPTION_ENABLED = "true"
$env:KMS_ENABLED = "true" # Optional
python run.py
cargo run -p myfsio-server --
# Bash
export ENCRYPTION_ENABLED=true
export KMS_ENABLED=true
python run.py</code></pre>
cargo run -p myfsio-server --</code></pre>
</li>
<li>
<strong>Configure bucket encryption:</strong> Navigate to your bucket → <strong>Properties</strong> tab → <strong>Default Encryption</strong> card → Click <strong>Enable Encryption</strong>.
@@ -1557,7 +1564,7 @@ curl "{{ api_base }}/my-bucket/secret.txt" \
<h3 class="h6 text-uppercase text-muted mt-4">How It Works</h3>
<p class="small text-muted mb-3">
Lifecycle rules run on a background timer (Python <code>threading.Timer</code>), not a system cronjob. The enforcement cycle triggers every <strong>3600 seconds (1 hour)</strong> by default. Each cycle scans all buckets with lifecycle configurations and applies matching rules.
Lifecycle rules run in a Tokio background task, not a system cronjob. The enforcement cycle triggers every <strong>3600 seconds (1 hour)</strong> by default. Each cycle scans all buckets with lifecycle configurations and applies matching rules.
</p>
<h3 class="h6 text-uppercase text-muted mt-4">Expiration Types</h3>
@@ -1639,7 +1646,7 @@ curl "{{ api_base }}/&lt;bucket&gt;?lifecycle" \
<h3 class="h6 text-uppercase text-muted mt-4">Enabling GC</h3>
<p class="small text-muted">Disabled by default. Enable via environment variable:</p>
<pre class="mb-3"><code class="language-bash">GC_ENABLED=true python run.py</code></pre>
<pre class="mb-3"><code class="language-bash">GC_ENABLED=true cargo run -p myfsio-server --</code></pre>
<h3 class="h6 text-uppercase text-muted mt-4">Configuration</h3>
<div class="table-responsive mb-3">
@@ -1742,7 +1749,7 @@ curl "{{ api_base }}/admin/gc/history?limit=10" \
<h3 class="h6 text-uppercase text-muted mt-4">Enabling Integrity Scanner</h3>
<p class="small text-muted">Disabled by default. Enable via environment variable:</p>
<pre class="mb-3"><code class="language-bash">INTEGRITY_ENABLED=true python run.py</code></pre>
<pre class="mb-3"><code class="language-bash">INTEGRITY_ENABLED=true cargo run -p myfsio-server --</code></pre>
<h3 class="h6 text-uppercase text-muted mt-4">Configuration</h3>
<div class="table-responsive mb-3">
@@ -1848,11 +1855,11 @@ curl "{{ api_base }}/admin/integrity/history?limit=10" \
<p class="small text-muted">Set the environment variable to opt-in:</p>
<pre class="mb-3"><code class="language-bash"># PowerShell
$env:METRICS_HISTORY_ENABLED = "true"
python run.py
cargo run -p myfsio-server --
# Bash
export METRICS_HISTORY_ENABLED=true
python run.py</code></pre>
cargo run -p myfsio-server --</code></pre>
<h3 class="h6 text-uppercase text-muted mt-4">Configuration Options</h3>
<div class="table-responsive mb-3">
@@ -1932,11 +1939,11 @@ curl -X PUT "{{ api_base | replace(from="/api", to="/ui") }}/metrics/settings" \
<p class="small text-muted">Set the environment variable to opt-in:</p>
<pre class="mb-3"><code class="language-bash"># PowerShell
$env:OPERATION_METRICS_ENABLED = "true"
python run.py
cargo run -p myfsio-server --
# Bash
export OPERATION_METRICS_ENABLED=true
python run.py</code></pre>
cargo run -p myfsio-server --</code></pre>
<h3 class="h6 text-uppercase text-muted mt-4">Configuration Options</h3>
<div class="table-responsive mb-3">
@@ -2751,8 +2758,8 @@ cp -r logs/ logs-backup/</code></pre>
<ol class="docs-steps mb-3">
<li><strong>Stop the service:</strong> <code>sudo systemctl stop myfsio</code> (or kill the process)</li>
<li><strong>Pull new version:</strong> <code>git pull origin main</code> or download the new binary</li>
<li><strong>Install dependencies:</strong> <code>pip install -r requirements.txt</code></li>
<li><strong>Validate config:</strong> <code>python run.py --check-config</code></li>
<li><strong>Build or install the new Rust binary:</strong> <code>cargo build --release -p myfsio-server</code></li>
<li><strong>Validate config:</strong> <code>./myfsio-server --check-config</code></li>
<li><strong>Start the service:</strong> <code>sudo systemctl start myfsio</code></li>
<li><strong>Verify:</strong> <code>curl http://localhost:5000/myfsio/health</code></li>
</ol>
@@ -2892,11 +2899,11 @@ POST /kms/generate-random # Generate random bytes</code></pre>
</div>
<div>
<div class="small text-uppercase text-muted">Logs</div>
<span class="text-muted small">logs/api.log · logs/ui.log</span>
<span class="text-muted small">stdout, journald, or container logs</span>
</div>
</div>
<div class="mt-4">
<p class="small text-muted mb-1">Need more automation? Extend <code>app/s3_api.py</code> or wrap <code>run_api.py</code> with gunicorn for production-style deployments.</p>
<p class="small text-muted mb-1">Need more automation? Integrate against the S3, admin, or KMS routes directly, or package the Rust binary behind systemd or containers for production deployments.</p>
</div>
</div>
</aside>

62
rust/myfsio-engine/crates/myfsio-server/templates/iam.html
View File

@@ -133,24 +133,8 @@
{% endif %}
<div class="row g-3">
{% for user in users %}
{% set_global is_admin = false %}
{% set_global is_expired = false %}
{% set_global is_expiring_soon = false %}
{% for policy in user.policies %}
{% if "iam:*" in policy.actions or "*" in policy.actions %}
{% set_global is_admin = true %}
{% endif %}
{% endfor %}
{% if user.expires_at %}
{% set exp_str = user.expires_at %}
{% if exp_str <= now_iso %}
{% set_global is_expired = true %}
{% elif exp_str <= soon_iso %}
{% set_global is_expiring_soon = true %}
{% endif %}
{% endif %}
<div class="col-md-6 col-xl-4 iam-user-item" data-display-name="{{ user.display_name|lower }}" data-access-key-filter="{{ user.access_key|lower }}">
<div class="card h-100 iam-user-card{% if is_admin %} iam-admin-card{% else %}{% endif %}">
<div class="col-md-6 col-xl-4 iam-user-item" data-user-id="{{ user.user_id }}" data-access-key="{{ user.access_key }}" data-display-name="{{ user.display_name|lower }}" data-access-key-filter="{{ user.access_key|lower }}">
<div class="card h-100 iam-user-card{% if user.is_admin %} iam-admin-card{% else %}{% endif %}">
<div class="card-body">
<div class="d-flex align-items-start justify-content-between mb-3">
<div class="d-flex align-items-center gap-3 min-width-0 overflow-hidden">
@@ -162,14 +146,14 @@
<div class="min-width-0">
<div class="d-flex align-items-center gap-2 mb-0">
<h6 class="fw-semibold mb-0 text-truncate" title="{{ user.display_name }}">{{ user.display_name }}</h6>
{% if is_admin %}
{% if user.is_admin %}
<span class="iam-role-badge iam-role-admin" data-role-badge>Admin</span>
{% else %}
<span class="iam-role-badge iam-role-user" data-role-badge>User</span>
{% endif %}
{% if is_expired %}
{% if user.is_expired %}
<span class="badge text-bg-danger" style="font-size: .65rem">Expired</span>
{% elif is_expiring_soon %}
{% elif user.is_expiring_soon %}
<span class="badge text-bg-warning" style="font-size: .65rem">Expiring soon</span>
{% endif %}
</div>
@@ -192,7 +176,7 @@
</button>
<ul class="dropdown-menu dropdown-menu-end">
<li>
<button class="dropdown-item" type="button" data-edit-user="{{ user.access_key }}" data-display-name="{{ user.display_name }}">
<button class="dropdown-item" type="button" data-edit-user data-user-id="{{ user.user_id }}" data-access-key="{{ user.access_key }}" data-display-name="{{ user.display_name }}">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16">
<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/>
</svg>
@@ -200,7 +184,7 @@
</button>
</li>
<li>
<button class="dropdown-item" type="button" data-expiry-user="{{ user.access_key }}" data-expires-at="{{ user.expires_at or "" }}">
<button class="dropdown-item" type="button" data-expiry-user data-user-id="{{ user.user_id }}" data-access-key="{{ user.access_key }}" data-expires-at="{{ user.expires_at | default(value="") }}">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16">
<path d="M8 3.5a.5.5 0 0 0-1 0V9a.5.5 0 0 0 .252.434l3.5 2a.5.5 0 0 0 .496-.868L8 8.71V3.5z"/>
<path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm7-8A7 7 0 1 1 1 8a7 7 0 0 1 14 0z"/>
@@ -209,7 +193,7 @@
</button>
</li>
<li>
<button class="dropdown-item" type="button" data-rotate-user="{{ user.access_key }}">
<button class="dropdown-item" type="button" data-rotate-user data-user-id="{{ user.user_id }}" data-access-key="{{ user.access_key }}">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16">
<path d="M11.534 7h3.932a.25.25 0 0 1 .192.41l-1.966 2.36a.25.25 0 0 1-.384 0l-1.966-2.36a.25.25 0 0 1 .192-.41zm-11 2h3.932a.25.25 0 0 0 .192-.41L2.692 6.23a.25.25 0 0 0-.384 0L.342 8.59A.25.25 0 0 0 .534 9z"/>
<path fill-rule="evenodd" d="M8 3c-1.552 0-2.94.707-3.857 1.818a.5.5 0 1 1-.771-.636A6.002 6.002 0 0 1 13.917 7H12.9A5.002 5.002 0 0 0 8 3zM3.1 9a5.002 5.002 0 0 0 8.757 2.182.5.5 0 1 1 .771.636A6.002 6.002 0 0 1 2.083 9H3.1z"/>
@@ -219,7 +203,7 @@
</li>
<li><hr class="dropdown-divider"></li>
<li>
<button class="dropdown-item text-danger" type="button" data-delete-user="{{ user.access_key }}">
<button class="dropdown-item text-danger" type="button" data-delete-user data-user-id="{{ user.user_id }}" data-access-key="{{ user.access_key }}">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16">
<path d="M5.5 5.5a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0v-6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0v-6a.5.5 0 0 1 .5-.5zm3 .5v6a.5.5 0 0 1-1 0v-6a.5.5 0 0 1 1 0z"/>
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
@@ -260,7 +244,7 @@
{% endfor %}
</div>
</div>
<button class="btn btn-outline-primary btn-sm w-100" type="button" data-policy-editor data-access-key="{{ user.access_key }}">
<button class="btn btn-outline-primary btn-sm w-100" type="button" data-policy-editor data-user-id="{{ user.user_id }}" data-access-key="{{ user.access_key }}">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
<path d="M8 4.754a3.246 3.246 0 1 0 0 6.492 3.246 3.246 0 0 0 0-6.492zM5.754 8a2.246 2.246 0 1 1 4.492 0 2.246 2.246 0 0 1-4.492 0z"/>
<path d="M9.796 1.343c-.527-1.79-3.065-1.79-3.592 0l-.094.319a.873.873 0 0 1-1.255.52l-.292-.16c-1.64-.892-3.433.902-2.54 2.541l.159.292a.873.873 0 0 1-.52 1.255l-.319.094c-1.79.527-1.79 3.065 0 3.592l.319.094a.873.873 0 0 1 .52 1.255l-.16.292c-.892 1.64.901 3.434 2.541 2.54l.292-.159a.873.873 0 0 1 1.255.52l.094.319c.527 1.79 3.065 1.79 3.592 0l.094-.319a.873.873 0 0 1 1.255-.52l.292.16c1.64.893 3.434-.902 2.54-2.541l-.159-.292a.873.873 0 0 1 .52-1.255l.319-.094c1.79-.527 1.79-3.065 0-3.592l-.319-.094a.873.873 0 0 1-.52-1.255l.16-.292c.893-1.64-.902-3.433-2.541-2.54l-.292.159a.873.873 0 0 1-1.255-.52l-.094-.319z"/>
@@ -313,7 +297,7 @@
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<form method="post" action="{{ url_for(endpoint="ui.create_iam_user") }}">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<div class="modal-body">
<div class="mb-3">
<label class="form-label fw-medium">Display Name</label>
@@ -393,11 +377,11 @@
<form
id="policyEditorForm"
method="post"
data-action-template="{{ url_for(endpoint="ui.update_iam_policies", access_key="ACCESS_KEY_PLACEHOLDER") }}"
data-action-template="{{ url_for(endpoint="ui.update_iam_policies", user_id="USER_ID_PLACEHOLDER") }}"
class="d-flex flex-column gap-3"
>
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" id="policyEditorUser" name="access_key" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<input type="hidden" id="policyEditorUserId" name="user_id" />
<div>
<label class="form-label fw-medium">Inline Policies (JSON array)</label>
@@ -440,7 +424,7 @@
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<form method="post" id="editUserForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<div class="modal-body">
<div class="mb-3">
<label class="form-label fw-medium">Display Name</label>
@@ -490,7 +474,7 @@
<div class="modal-footer">
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
<form method="post" id="deleteUserForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<button class="btn btn-danger" type="submit">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
@@ -575,7 +559,7 @@
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<form method="post" id="expiryForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<div class="modal-body">
<p class="text-muted small mb-3">Set expiration for <code id="expiryUserLabel"></code></p>
<div class="mb-3">
@@ -618,14 +602,14 @@
users: JSON.parse(document.getElementById('iamUsersJson').textContent || '[]'),
currentUserKey: {{ principal.access_key | json_encode | safe }},
iamLocked: {{ iam_locked | json_encode | safe }},
csrfToken: "{{ csrf_token() }}",
csrfToken: "{{ csrf_token_value }}",
endpoints: {
createUser: "{{ url_for(endpoint="ui.create_iam_user") }}",
updateUser: "{{ url_for(endpoint="ui.update_iam_user", access_key="ACCESS_KEY") }}",
deleteUser: "{{ url_for(endpoint="ui.delete_iam_user", access_key="ACCESS_KEY") }}",
updatePolicies: "{{ url_for(endpoint="ui.update_iam_policies", access_key="ACCESS_KEY") }}",
rotateSecret: "{{ url_for(endpoint="ui.rotate_iam_secret", access_key="ACCESS_KEY") }}",
updateExpiry: "{{ url_for(endpoint="ui.update_iam_expiry", access_key="ACCESS_KEY") }}"
updateUser: "{{ url_for(endpoint="ui.update_iam_user", user_id="USER_ID") }}",
deleteUser: "{{ url_for(endpoint="ui.delete_iam_user", user_id="USER_ID") }}",
updatePolicies: "{{ url_for(endpoint="ui.update_iam_policies", user_id="USER_ID") }}",
rotateSecret: "{{ url_for(endpoint="ui.rotate_iam_secret", user_id="USER_ID") }}",
updateExpiry: "{{ url_for(endpoint="ui.update_iam_expiry", user_id="USER_ID") }}"
}
});
</script>

2
rust/myfsio-engine/crates/myfsio-server/templates/login.html
View File

@@ -41,7 +41,7 @@
<h2 class="h4 mb-1 d-none d-lg-block">Sign in</h2>
<p class="text-muted mb-4 d-none d-lg-block">Enter your credentials to continue</p>
<form method="post" action="{{ url_for(endpoint="ui.login") }}">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}" />
<div class="mb-3">
<label class="form-label fw-medium">Access key</label>
<div class="input-group">

2
rust/myfsio-engine/crates/myfsio-server/templates/replication_wizard.html
View File

@@ -78,7 +78,7 @@
<div class="card-body px-4 pb-4">
{% if buckets %}
<form method="POST" action="{{ url_for(endpoint="ui.create_peer_replication_rules", site_id=peer.site_id) }}">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}"/>
<div class="mb-4">
<label for="mode" class="form-label fw-medium">Replication Mode</label>

18
rust/myfsio-engine/crates/myfsio-server/templates/sites.html
View File

@@ -43,18 +43,18 @@
</div>
<div class="card-body px-4 pb-4">
<form method="POST" action="{{ url_for(endpoint="ui.update_local_site") }}" id="localSiteForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}"/>
<div class="mb-3">
<label for="site_id" class="form-label fw-medium">Site ID</label>
<input type="text" class="form-control" id="site_id" name="site_id" required
value="{% if local_site %}{{ local_site.site_id }}{% else %}{{ config_site_id or "" }}{% endif %}"
value="{% if local_site %}{{ local_site.site_id }}{% else %}{{ config_site_id | default(value="") }}{% endif %}"
placeholder="us-west-1">
<div class="form-text">Unique identifier for this site</div>
</div>
<div class="mb-3">
<label for="endpoint" class="form-label fw-medium">Endpoint URL</label>
<input type="url" class="form-control" id="endpoint" name="endpoint"
value="{% if local_site %}{{ local_site.endpoint }}{% else %}{{ config_site_endpoint or "" }}{% endif %}"
value="{% if local_site %}{{ local_site.endpoint }}{% else %}{{ config_site_endpoint | default(value="") }}{% endif %}"
placeholder="https://s3.us-west-1.example.com">
<div class="form-text">Public URL for this site</div>
</div>
@@ -109,7 +109,7 @@
<div class="collapse" id="addPeerCollapse">
<div class="card-body px-4 pb-4">
<form method="POST" action="{{ url_for(endpoint="ui.add_peer_site") }}" id="addPeerForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}"/>
<div class="mb-3">
<label for="peer_site_id" class="form-label fw-medium">Site ID</label>
<input type="text" class="form-control" id="peer_site_id" name="site_id" required placeholder="us-east-1">
@@ -198,7 +198,7 @@
<tr data-site-id="{{ peer.site_id }}">
<td class="text-center">
<span class="peer-health-status" data-site-id="{{ peer.site_id }}"
data-last-checked="{{ peer.last_health_check or "" }}"
data-last-checked="{{ peer.last_health_check | default(value="") }}"
title="{% if peer.is_healthy == true %}Healthy{% elif peer.is_healthy == false %}Unhealthy{% else %}Not checked{% endif %}{% if peer.last_health_check %} (checked {{ peer.last_health_check }}){% endif %}"
style="cursor: help;">
{% if peer.is_healthy == true %}
@@ -277,7 +277,7 @@
data-region="{{ peer.region }}"
data-priority="{{ peer.priority }}"
data-display-name="{{ peer.display_name }}"
data-connection-id="{{ peer.connection_id or "" }}"
data-connection-id="{{ peer.connection_id | default(value="") }}"
title="Edit peer">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/>
@@ -381,7 +381,7 @@
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<form method="POST" id="editPeerForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}"/>
<div class="modal-body">
<div class="mb-3">
<label class="form-label fw-medium">Site ID</label>
@@ -454,7 +454,7 @@
<div class="modal-footer">
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
<form method="POST" id="deletePeerForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}"/>
<button type="submit" class="btn btn-danger">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
@@ -775,7 +775,7 @@
var blockingErrors = ['NO_CONNECTION', 'CONNECTION_NOT_FOUND', 'REMOTE_UNREACHABLE', 'ENDPOINT_NOT_ALLOWED'];
var hasBlockingError = data.issues && data.issues.some(function(i) { return blockingErrors.indexOf(i.code) !== -1; });
if (!hasBlockingError) {
wizardLink.href = '/ui/sites/peers/' + encodeURIComponent(siteId) + '/replication-wizard';
wizardLink.href = '/ui/replication/new?site_id=' + encodeURIComponent(siteId);
wizardLink.classList.remove('d-none');
}
}

43
rust/myfsio-engine/crates/myfsio-server/templates/system.html
View File

@@ -37,13 +37,8 @@
<tr><td class="text-muted" style="width:40%">Version</td><td class="fw-medium">{{ app_version }}</td></tr>
<tr><td class="text-muted">Storage Root</td><td><code>{{ storage_root }}</code></td></tr>
<tr><td class="text-muted">Platform</td><td>{{ platform }}</td></tr>
<tr><td class="text-muted">Python</td><td>{{ python_version }}</td></tr>
<tr><td class="text-muted">Rust Extension</td><td>
{% if has_rust %}
<span class="badge bg-success bg-opacity-10 text-success">Loaded</span>
{% else %}
<span class="badge bg-secondary bg-opacity-10 text-secondary">Not loaded</span>
{% endif %}
<tr><td class="text-muted">Engine</td><td>
<span class="badge bg-success bg-opacity-10 text-success">Rust (native)</span>
</td></tr>
</tbody>
</table>
@@ -420,7 +415,8 @@
function _gcRefreshHistory() {
fetch('{{ url_for(endpoint="ui.system_gc_history") }}?limit=10', {
headers: {'X-CSRFToken': csrfToken}
headers: {'X-CSRFToken': csrfToken},
cache: 'no-store'
})
.then(function (r) { return r.json(); })
.then(function (hist) {
@@ -459,7 +455,8 @@
function _integrityRefreshHistory() {
fetch('{{ url_for(endpoint="ui.system_integrity_history") }}?limit=10', {
headers: {'X-CSRFToken': csrfToken}
headers: {'X-CSRFToken': csrfToken},
cache: 'no-store'
})
.then(function (r) { return r.json(); })
.then(function (hist) {
@@ -555,7 +552,8 @@
function _gcPoll() {
fetch('{{ url_for(endpoint="ui.system_gc_status") }}', {
headers: {'X-CSRFToken': csrfToken}
headers: {'X-CSRFToken': csrfToken},
cache: 'no-store'
})
.then(function (r) { return r.json(); })
.then(function (status) {
@@ -567,7 +565,8 @@
_gcSetScanning(false);
_gcRefreshHistory();
fetch('{{ url_for(endpoint="ui.system_gc_history") }}?limit=1', {
headers: {'X-CSRFToken': csrfToken}
headers: {'X-CSRFToken': csrfToken},
cache: 'no-store'
})
.then(function (r) { return r.json(); })
.then(function (hist) {
@@ -608,7 +607,13 @@
body.textContent = data.error;
return;
}
_gcPollTimer = setTimeout(_gcPoll, 2000);
if (data.status === 'started' || data.scanning === true || data.running === true) {
_gcPollTimer = setTimeout(_gcPoll, 2000);
return;
}
_gcSetScanning(false);
_gcShowResult(data, dryRun);
_gcRefreshHistory();
})
.catch(function (err) {
_gcSetScanning(false);
@@ -685,7 +690,8 @@
function _integrityPoll() {
fetch('{{ url_for(endpoint="ui.system_integrity_status") }}', {
headers: {'X-CSRFToken': csrfToken}
headers: {'X-CSRFToken': csrfToken},
cache: 'no-store'
})
.then(function (r) { return r.json(); })
.then(function (status) {
@@ -697,7 +703,8 @@
_integritySetScanning(false);
_integrityRefreshHistory();
fetch('{{ url_for(endpoint="ui.system_integrity_history") }}?limit=1', {
headers: {'X-CSRFToken': csrfToken}
headers: {'X-CSRFToken': csrfToken},
cache: 'no-store'
})
.then(function (r) { return r.json(); })
.then(function (hist) {
@@ -738,7 +745,13 @@
body.textContent = data.error;
return;
}
_integrityPollTimer = setTimeout(_integrityPoll, 2000);
if (data.status === 'started' || data.scanning === true || data.running === true) {
_integrityPollTimer = setTimeout(_integrityPoll, 2000);
return;
}
_integritySetScanning(false);
_integrityShowResult(data, dryRun, autoHeal);
_integrityRefreshHistory();
})
.catch(function (err) {
_integritySetScanning(false);

6
rust/myfsio-engine/crates/myfsio-server/templates/website_domains.html
View File

@@ -35,7 +35,7 @@
</div>
<div class="card-body px-4 pb-4">
<form method="POST" action="{{ url_for(endpoint="ui.create_website_domain") }}" id="createDomainForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}"/>
<div class="mb-3">
<label for="domain" class="form-label fw-medium">Domain</label>
<input type="text" class="form-control" id="domain" name="domain" required
@@ -204,7 +204,7 @@
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<form method="POST" id="editDomainForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}"/>
<div class="modal-body">
<div class="mb-3">
<label class="form-label fw-medium">Domain</label>
@@ -241,7 +241,7 @@
<div class="modal-dialog modal-dialog-centered">
<div class="modal-content">
<form method="POST" id="deleteDomainForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<input type="hidden" name="csrf_token" value="{{ csrf_token_value }}"/>
<div class="modal-header border-0 pb-0">
<h5 class="modal-title fw-semibold">
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-danger" viewBox="0 0 16 16">

1364
rust/myfsio-engine/crates/myfsio-server/tests/integration.rs
View File

File diff suppressed because it is too large Load Diff

343
rust/myfsio-engine/crates/myfsio-server/tests/template_render.rs Normal file
View File

@@ -0,0 +1,343 @@
use std::error::Error as StdError;
use std::path::PathBuf;
use myfsio_server::templates::TemplateEngine;
use serde_json::{json, Value};
use tera::Context;
fn engine() -> TemplateEngine {
let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
path.push("templates");
path.push("*.html");
let glob = path.to_string_lossy().replace('\\', "/");
let engine = TemplateEngine::new(&glob).expect("template parse");
myfsio_server::handlers::ui_pages::register_ui_endpoints(&engine);
engine
}
fn base_ctx() -> Context {
let mut ctx = Context::new();
ctx.insert("csrf_token_value", &"test-csrf");
ctx.insert("is_authenticated", &true);
ctx.insert("current_user", &"test");
ctx.insert("current_user_display_name", &"Test User");
ctx.insert("current_endpoint", &"");
ctx.insert("request_args", &serde_json::json!({}));
ctx.insert(
"principal",
&json!({
"access_key": "AKIATESTTEST",
"user_id": "u-test",
"display_name": "Test Admin",
"is_admin": true
}),
);
ctx.insert("can_manage_iam", &true);
ctx.insert("can_manage_replication", &true);
ctx.insert("can_manage_sites", &true);
ctx.insert("can_manage_encryption", &false);
ctx.insert("website_hosting_nav", &false);
ctx.insert("encryption_enabled", &false);
ctx.insert("kms_enabled", &false);
ctx.insert("flashed_messages", &Vec::<Value>::new());
ctx.insert("null", &Value::Null);
ctx.insert("none", &Value::Null);
ctx
}
fn format_err(e: tera::Error) -> String {
let mut out = format!("{}", e);
let mut src = StdError::source(&e);
while let Some(s) = src {
out.push_str("\n caused by: ");
out.push_str(&s.to_string());
src = s.source();
}
out
}
fn render_or_panic(tmpl: &str, ctx: &Context) {
let e = engine();
match e.render(tmpl, ctx) {
Ok(_) => {}
Err(err) => panic!("{} failed:\n{}", tmpl, format_err(err)),
}
}
fn render_to_string_or_panic(tmpl: &str, ctx: &Context) -> String {
let e = engine();
match e.render(tmpl, ctx) {
Ok(rendered) => rendered,
Err(err) => panic!("{} failed:\n{}", tmpl, format_err(err)),
}
}
#[test]
fn render_buckets() {
let mut ctx = base_ctx();
ctx.insert(
"buckets",
&json!([{
"meta": {"name": "b1", "creation_date": "2024-01-01T00:00:00Z"},
"summary": {"human_size": "0 B", "objects": 0},
"detail_url": "/ui/buckets/b1",
"access_badge": "bg-secondary",
"access_label": "Private"
}]),
);
render_or_panic("buckets.html", &ctx);
}
#[test]
fn render_connections() {
let mut ctx = base_ctx();
ctx.insert(
"connections",
&json!([{
"id": "c1",
"name": "Prod",
"endpoint_url": "https://s3.example.com",
"region": "us-east-1",
"access_key": "AKIAEXAMPLEKEY12345"
}]),
);
render_or_panic("connections.html", &ctx);
}
#[test]
fn render_iam() {
let mut ctx = base_ctx();
ctx.insert(
"users",
&json!([{
"user_id": "u-1",
"access_key": "AKIA1",
"display_name": "Alice",
"enabled": true,
"is_enabled": true,
"expires_at": "2026-04-21T00:00:00Z",
"is_admin": true,
"is_expired": false,
"is_expiring_soon": true,
"access_keys": [{"access_key": "AKIA1", "status": "active", "created_at": "2024-01-01"}],
"policy_count": 1,
"policies": [{"bucket": "*", "actions": ["*"], "prefix": "*"}]
}]),
);
ctx.insert("iam_locked", &false);
ctx.insert("locked_reason", &"");
ctx.insert("iam_disabled", &false);
ctx.insert("all_buckets", &Vec::<String>::new());
ctx.insert("disclosed_secret", &Value::Null);
ctx.insert("config_document", &"");
ctx.insert("config_summary", &json!({"user_count": 1}));
let rendered = render_to_string_or_panic("iam.html", &ctx);
assert!(rendered.contains("data-user-id=\"u-1\""));
assert!(rendered.contains("data-access-key=\"AKIA1\""));
assert!(rendered.contains("Expiring soon"));
}
#[test]
fn render_metrics() {
let mut ctx = base_ctx();
ctx.insert("metrics_enabled", &false);
ctx.insert("metrics_history_enabled", &false);
ctx.insert("operation_metrics_enabled", &false);
ctx.insert("history", &Vec::<Value>::new());
ctx.insert("operation_metrics", &Vec::<Value>::new());
ctx.insert("cpu_percent", &0);
ctx.insert(
"memory",
&json!({ "percent": 0, "total": "0 B", "used": "0 B" }),
);
ctx.insert(
"disk",
&json!({ "percent": 0, "free": "0 B", "total": "0 B" }),
);
ctx.insert(
"app",
&json!({
"buckets": 0, "objects": 0, "storage_used": "0 B",
"uptime_days": 0, "versions": 0
}),
);
ctx.insert("has_issues", &false);
ctx.insert(
"summary",
&json!({
"app": {"buckets": 0, "objects": 0, "storage_used": "0 B", "uptime_days": 0, "versions": 0},
"cpu_percent": 0,
"disk": {"free": 0, "percent": 0, "total": 0},
"memory": {"percent": 0, "total": 0, "used": 0},
"has_issues": false
}),
);
render_or_panic("metrics.html", &ctx);
}
#[test]
fn render_system() {
let mut ctx = base_ctx();
ctx.insert("app_version", &"0.1.0");
ctx.insert("display_timezone", &"UTC");
ctx.insert("platform", &"linux");
ctx.insert("python_version", &"n/a");
ctx.insert("storage_root", &"/tmp/data");
ctx.insert("has_rust", &true);
ctx.insert("total_issues", &0);
ctx.insert("features", &Vec::<Value>::new());
ctx.insert("gc_history", &Vec::<Value>::new());
ctx.insert("integrity_history", &Vec::<Value>::new());
ctx.insert(
"gc_status",
&json!({
"dry_run": false, "enabled": false, "interval_hours": 6,
"lock_file_max_age_hours": 1, "multipart_max_age_days": 7,
"scanning": false, "temp_file_max_age_hours": 24
}),
);
ctx.insert(
"integrity_status",
&json!({
"auto_heal": false, "batch_size": 100, "dry_run": false,
"enabled": false, "interval_hours": 24, "scanning": false
}),
);
render_or_panic("system.html", &ctx);
}
#[test]
fn render_sites() {
let mut ctx = base_ctx();
ctx.insert("local_site", &Value::Null);
ctx.insert("peers", &Vec::<Value>::new());
ctx.insert("peers_with_stats", &Vec::<Value>::new());
ctx.insert("connections", &Vec::<Value>::new());
ctx.insert("config_site_id", &"");
ctx.insert("config_site_endpoint", &"");
ctx.insert("config_site_region", &"us-east-1");
render_or_panic("sites.html", &ctx);
}
#[test]
fn render_website_domains() {
let mut ctx = base_ctx();
ctx.insert("mappings", &Vec::<Value>::new());
ctx.insert("buckets", &Vec::<String>::new());
render_or_panic("website_domains.html", &ctx);
}
#[test]
fn render_replication_wizard() {
let mut ctx = base_ctx();
ctx.insert("connections", &Vec::<Value>::new());
ctx.insert("local_site", &Value::Null);
ctx.insert("peers", &Vec::<Value>::new());
ctx.insert("buckets", &Vec::<Value>::new());
ctx.insert(
"peer",
&json!({
"site_id": "peer-1",
"display_name": "Peer One",
"endpoint": "https://peer.example.com",
"region": "us-east-1"
}),
);
ctx.insert(
"connection",
&json!({
"id": "c1", "name": "Prod",
"endpoint_url": "https://s3.example.com",
"region": "us-east-1",
"access_key": "AKIA"
}),
);
render_or_panic("replication_wizard.html", &ctx);
}
#[test]
fn render_docs() {
let mut ctx = base_ctx();
ctx.insert("api_base", &"http://127.0.0.1:9000");
ctx.insert("api_host", &"127.0.0.1:9000");
render_or_panic("docs.html", &ctx);
}
#[test]
fn render_bucket_detail() {
let mut ctx = base_ctx();
ctx.insert("bucket_name", &"my-bucket");
ctx.insert(
"bucket",
&json!({
"name": "my-bucket",
"creation_date": "2024-01-01T00:00:00Z"
}),
);
ctx.insert("objects", &Vec::<Value>::new());
ctx.insert("prefixes", &Vec::<Value>::new());
ctx.insert("total_objects", &0);
ctx.insert("total_bytes", &0);
ctx.insert("current_objects", &0);
ctx.insert("current_bytes", &0);
ctx.insert("version_count", &0);
ctx.insert("version_bytes", &0);
ctx.insert("max_objects", &Value::Null);
ctx.insert("max_bytes", &Value::Null);
ctx.insert("obj_pct", &0);
ctx.insert("bytes_pct", &0);
ctx.insert("has_quota", &false);
ctx.insert("versioning_enabled", &false);
ctx.insert("versioning_status", &"Disabled");
ctx.insert("encryption_config", &json!({"Rules": []}));
ctx.insert("enc_rules", &Vec::<Value>::new());
ctx.insert("enc_algorithm", &"");
ctx.insert("enc_kms_key", &"");
ctx.insert("replication_rules", &Vec::<Value>::new());
ctx.insert("replication_rule", &Value::Null);
ctx.insert("website_config", &Value::Null);
ctx.insert("bucket_policy", &"");
ctx.insert("bucket_policy_text", &"");
ctx.insert("connections", &Vec::<Value>::new());
ctx.insert("current_prefix", &"");
ctx.insert("parent_prefix", &"");
ctx.insert("has_more", &false);
ctx.insert("next_token", &"");
ctx.insert("active_tab", &"objects");
ctx.insert("multipart_uploads", &Vec::<Value>::new());
ctx.insert("target_conn", &Value::Null);
ctx.insert("target_conn_name", &"");
ctx.insert("preset_choice", &"");
ctx.insert("default_policy", &"");
ctx.insert("can_manage_cors", &true);
ctx.insert("can_manage_lifecycle", &true);
ctx.insert("can_manage_quota", &true);
ctx.insert("can_manage_versioning", &true);
ctx.insert("can_manage_website", &true);
ctx.insert("can_edit_policy", &true);
ctx.insert("is_replication_admin", &true);
ctx.insert("lifecycle_enabled", &false);
ctx.insert("site_sync_enabled", &false);
ctx.insert("website_hosting_enabled", &false);
ctx.insert("website_domains", &Vec::<Value>::new());
ctx.insert("kms_keys", &Vec::<Value>::new());
ctx.insert(
"bucket_stats",
&json!({
"bytes": 0, "objects": 0, "total_bytes": 0, "total_objects": 0,
"version_bytes": 0, "version_count": 0
}),
);
ctx.insert(
"bucket_quota",
&json!({ "max_bytes": null, "max_objects": null }),
);
ctx.insert("buckets_for_copy_url", &"");
ctx.insert("acl_url", &"");
ctx.insert("cors_url", &"");
ctx.insert("folders_url", &"");
ctx.insert("lifecycle_url", &"");
ctx.insert("objects_api_url", &"");
ctx.insert("objects_stream_url", &"");
render_or_panic("bucket_detail.html", &ctx);
}

4
rust/myfsio-engine/crates/myfsio-storage/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "myfsio-storage"
version = "0.1.0"
edition = "2021"
version.workspace = true
edition.workspace = true
[dependencies]
myfsio-common = { path = "../myfsio-common" }

14
rust/myfsio-engine/crates/myfsio-storage/src/error.rs
View File

@@ -40,18 +40,20 @@ impl From<StorageError> for S3Error {
.with_resource(format!("/{}", name))
}
StorageError::BucketNotEmpty(name) => {
S3Error::from_code(S3ErrorCode::BucketNotEmpty)
.with_resource(format!("/{}", name))
S3Error::from_code(S3ErrorCode::BucketNotEmpty).with_resource(format!("/{}", name))
}
StorageError::ObjectNotFound { bucket, key } => {
S3Error::from_code(S3ErrorCode::NoSuchKey)
.with_resource(format!("/{}/{}", bucket, key))
}
StorageError::InvalidBucketName(msg) => S3Error::new(S3ErrorCode::InvalidBucketName, msg),
StorageError::InvalidObjectKey(msg) => S3Error::new(S3ErrorCode::InvalidKey, msg),
StorageError::UploadNotFound(id) => {
S3Error::new(S3ErrorCode::NoSuchUpload, format!("Upload {} not found", id))
StorageError::InvalidBucketName(msg) => {
S3Error::new(S3ErrorCode::InvalidBucketName, msg)
}
StorageError::InvalidObjectKey(msg) => S3Error::new(S3ErrorCode::InvalidKey, msg),
StorageError::UploadNotFound(id) => S3Error::new(
S3ErrorCode::NoSuchUpload,
format!("Upload {} not found", id),
),
StorageError::QuotaExceeded(msg) => S3Error::new(S3ErrorCode::QuotaExceeded, msg),
StorageError::InvalidRange => S3Error::from_code(S3ErrorCode::InvalidRange),
StorageError::Io(e) => S3Error::new(S3ErrorCode::InternalError, e.to_string()),

141
rust/myfsio-engine/crates/myfsio-storage/src/fs_backend.rs
View File

@@ -76,7 +76,8 @@ impl FsStorageBackend {
}
fn bucket_versions_root(&self, bucket_name: &str) -> PathBuf {
self.system_bucket_root(bucket_name).join(BUCKET_VERSIONS_DIR)
self.system_bucket_root(bucket_name)
.join(BUCKET_VERSIONS_DIR)
}
fn multipart_root(&self) -> PathBuf {
@@ -142,7 +143,8 @@ impl FsStorageBackend {
}
fn bucket_config_path(&self, bucket_name: &str) -> PathBuf {
self.system_bucket_root(bucket_name).join(BUCKET_CONFIG_FILE)
self.system_bucket_root(bucket_name)
.join(BUCKET_CONFIG_FILE)
}
fn version_dir(&self, bucket_name: &str, key: &str) -> PathBuf {
@@ -210,11 +212,7 @@ impl FsStorageBackend {
.and_then(|index| {
index.get(&entry_name).and_then(|v| {
if let Value::Object(map) = v {
Some(
map.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect(),
)
Some(map.iter().map(|(k, v)| (k.clone(), v.clone())).collect())
} else {
None
}
@@ -319,9 +317,7 @@ impl FsStorageBackend {
if let Some(Value::Object(meta)) = payload.get("metadata") {
return meta
.iter()
.filter_map(|(k, v)| {
v.as_str().map(|s| (k.clone(), s.to_string()))
})
.filter_map(|(k, v)| v.as_str().map(|s| (k.clone(), s.to_string())))
.collect();
}
}
@@ -424,9 +420,8 @@ impl FsStorageBackend {
let has_objects = Self::dir_has_files(bucket_path, Some(INTERNAL_FOLDERS));
let has_versions = Self::dir_has_files(&self.bucket_versions_root(&bucket_name), None)
|| Self::dir_has_files(&self.legacy_versions_root(&bucket_name), None);
let has_multipart =
Self::dir_has_files(&self.multipart_bucket_root(&bucket_name), None)
|| Self::dir_has_files(&self.legacy_multipart_root(&bucket_name), None);
let has_multipart = Self::dir_has_files(&self.multipart_bucket_root(&bucket_name), None)
|| Self::dir_has_files(&self.legacy_multipart_root(&bucket_name), None);
(has_objects, has_versions, has_multipart)
}
@@ -915,10 +910,46 @@ impl FsStorageBackend {
}
let is_overwrite = destination.exists();
let existing_size = if is_overwrite {
std::fs::metadata(&destination)
.map(|m| m.len())
.unwrap_or(0)
} else {
0
};
let bucket_config = self.read_bucket_config_sync(bucket_name);
if let Some(quota) = bucket_config.quota.as_ref() {
self.stats_cache.remove(bucket_name);
let stats = self.bucket_stats_sync(bucket_name)?;
let added_bytes = new_size.saturating_sub(existing_size);
let added_objects: u64 = if is_overwrite { 0 } else { 1 };
if let Some(max_bytes) = quota.max_bytes {
let projected = stats.total_bytes().saturating_add(added_bytes);
if projected > max_bytes {
let _ = std::fs::remove_file(tmp_path);
return Err(StorageError::QuotaExceeded(format!(
"Quota exceeded: adding {} bytes would result in {} bytes, exceeding limit of {} bytes",
added_bytes, projected, max_bytes
)));
}
}
if let Some(max_objects) = quota.max_objects {
let projected = stats.total_objects().saturating_add(added_objects);
if projected > max_objects {
let _ = std::fs::remove_file(tmp_path);
return Err(StorageError::QuotaExceeded(format!(
"Quota exceeded: adding {} objects would result in {} objects, exceeding limit of {} objects",
added_objects, projected, max_objects
)));
}
}
}
let lock_dir = self.system_bucket_root(bucket_name).join("locks");
std::fs::create_dir_all(&lock_dir).map_err(StorageError::Io)?;
let versioning_enabled = self.read_bucket_config_sync(bucket_name).versioning_enabled;
let versioning_enabled = bucket_config.versioning_enabled;
if versioning_enabled && is_overwrite {
self.archive_current_version_sync(bucket_name, key, "overwrite")
.map_err(StorageError::Io)?;
@@ -929,6 +960,8 @@ impl FsStorageBackend {
StorageError::Io(e)
})?;
self.stats_cache.remove(bucket_name);
let file_meta = std::fs::metadata(&destination).map_err(StorageError::Io)?;
let mtime = file_meta
.modified()
@@ -1161,7 +1194,9 @@ impl crate::traits::StorageEngine for FsStorageBackend {
.filter(|(k, _)| !k.starts_with("__"))
.collect();
let file = tokio::fs::File::open(&path).await.map_err(StorageError::Io)?;
let file = tokio::fs::File::open(&path)
.await
.map_err(StorageError::Io)?;
let stream: AsyncReadStream = Box::pin(file);
Ok((obj, stream))
}
@@ -1313,8 +1348,7 @@ impl crate::traits::StorageEngine for FsStorageBackend {
});
let manifest_path = upload_dir.join(MANIFEST_FILE);
Self::atomic_write_json_sync(&manifest_path, &manifest, true)
.map_err(StorageError::Io)?;
Self::atomic_write_json_sync(&manifest_path, &manifest, true).map_err(StorageError::Io)?;
Ok(upload_id)
}
@@ -1367,8 +1401,7 @@ impl crate::traits::StorageEngine for FsStorageBackend {
let lock = self.get_meta_index_lock(&lock_path.to_string_lossy());
let _guard = lock.lock();
let manifest_content =
std::fs::read_to_string(&manifest_path).map_err(StorageError::Io)?;
let manifest_content = std::fs::read_to_string(&manifest_path).map_err(StorageError::Io)?;
let mut manifest: Value =
serde_json::from_str(&manifest_content).map_err(StorageError::Json)?;
@@ -1383,8 +1416,7 @@ impl crate::traits::StorageEngine for FsStorageBackend {
);
}
Self::atomic_write_json_sync(&manifest_path, &manifest, true)
.map_err(StorageError::Io)?;
Self::atomic_write_json_sync(&manifest_path, &manifest, true).map_err(StorageError::Io)?;
Ok(etag)
}
@@ -1421,7 +1453,10 @@ impl crate::traits::StorageEngine for FsStorageBackend {
.map(|d| d.as_secs_f64())
.unwrap_or(0.0);
let last_modified = Utc
.timestamp_opt(src_mtime as i64, ((src_mtime % 1.0) * 1_000_000_000.0) as u32)
.timestamp_opt(
src_mtime as i64,
((src_mtime % 1.0) * 1_000_000_000.0) as u32,
)
.single()
.unwrap_or_else(Utc::now);
@@ -1490,8 +1525,7 @@ impl crate::traits::StorageEngine for FsStorageBackend {
let lock = self.get_meta_index_lock(&lock_path.to_string_lossy());
let _guard = lock.lock();
let manifest_content =
std::fs::read_to_string(&manifest_path).map_err(StorageError::Io)?;
let manifest_content = std::fs::read_to_string(&manifest_path).map_err(StorageError::Io)?;
let mut manifest: Value =
serde_json::from_str(&manifest_content).map_err(StorageError::Json)?;
@@ -1506,8 +1540,7 @@ impl crate::traits::StorageEngine for FsStorageBackend {
);
}
Self::atomic_write_json_sync(&manifest_path, &manifest, true)
.map_err(StorageError::Io)?;
Self::atomic_write_json_sync(&manifest_path, &manifest, true).map_err(StorageError::Io)?;
Ok((etag, last_modified))
}
@@ -1524,8 +1557,7 @@ impl crate::traits::StorageEngine for FsStorageBackend {
return Err(StorageError::UploadNotFound(upload_id.to_string()));
}
let manifest_content =
std::fs::read_to_string(&manifest_path).map_err(StorageError::Io)?;
let manifest_content = std::fs::read_to_string(&manifest_path).map_err(StorageError::Io)?;
let manifest: Value =
serde_json::from_str(&manifest_content).map_err(StorageError::Json)?;
@@ -1590,7 +1622,14 @@ impl crate::traits::StorageEngine for FsStorageBackend {
composite_hasher.update(&md5_digest_concat);
let etag = format!("{:x}-{}", composite_hasher.finalize(), part_count);
let result = self.finalize_put_sync(bucket, &object_key, &tmp_path, etag, total_size, Some(metadata))?;
let result = self.finalize_put_sync(
bucket,
&object_key,
&tmp_path,
etag,
total_size,
Some(metadata),
)?;
let _ = std::fs::remove_dir_all(&upload_dir);
@@ -1763,15 +1802,14 @@ impl crate::traits::StorageEngine for FsStorageBackend {
Ok(versions)
}
async fn get_object_tags(
&self,
bucket: &str,
key: &str,
) -> StorageResult<Vec<Tag>> {
async fn get_object_tags(&self, bucket: &str, key: &str) -> StorageResult<Vec<Tag>> {
self.require_bucket(bucket)?;
let obj_path = self.object_path(bucket, key)?;
if !obj_path.exists() {
return Err(StorageError::ObjectNotFound { bucket: bucket.to_string(), key: key.to_string() });
return Err(StorageError::ObjectNotFound {
bucket: bucket.to_string(),
key: key.to_string(),
});
}
let entry = self.read_index_entry_sync(bucket, key);
@@ -1785,16 +1823,14 @@ impl crate::traits::StorageEngine for FsStorageBackend {
Ok(Vec::new())
}
async fn set_object_tags(
&self,
bucket: &str,
key: &str,
tags: &[Tag],
) -> StorageResult<()> {
async fn set_object_tags(&self, bucket: &str, key: &str, tags: &[Tag]) -> StorageResult<()> {
self.require_bucket(bucket)?;
let obj_path = self.object_path(bucket, key)?;
if !obj_path.exists() {
return Err(StorageError::ObjectNotFound { bucket: bucket.to_string(), key: key.to_string() });
return Err(StorageError::ObjectNotFound {
bucket: bucket.to_string(),
key: key.to_string(),
});
}
let mut entry = self.read_index_entry_sync(bucket, key).unwrap_or_default();
@@ -1812,11 +1848,7 @@ impl crate::traits::StorageEngine for FsStorageBackend {
Ok(())
}
async fn delete_object_tags(
&self,
bucket: &str,
key: &str,
) -> StorageResult<()> {
async fn delete_object_tags(&self, bucket: &str, key: &str) -> StorageResult<()> {
self.set_object_tags(bucket, key, &[]).await
}
}
@@ -1883,7 +1915,10 @@ mod tests {
assert_eq!(meta.size, 11);
assert!(meta.etag.is_some());
let (obj, mut stream) = backend.get_object("test-bucket", "greeting.txt").await.unwrap();
let (obj, mut stream) = backend
.get_object("test-bucket", "greeting.txt")
.await
.unwrap();
assert_eq!(obj.size, 11);
let mut buf = Vec::new();
stream.read_to_end(&mut buf).await.unwrap();
@@ -1901,7 +1936,10 @@ mod tests {
.await
.unwrap();
let meta = backend.head_object("test-bucket", "file.txt").await.unwrap();
let meta = backend
.head_object("test-bucket", "file.txt")
.await
.unwrap();
assert_eq!(meta.size, 9);
assert!(meta.etag.is_some());
}
@@ -1917,7 +1955,10 @@ mod tests {
.await
.unwrap();
backend.delete_object("test-bucket", "file.txt").await.unwrap();
backend
.delete_object("test-bucket", "file.txt")
.await
.unwrap();
let result = backend.head_object("test-bucket", "file.txt").await;
assert!(result.is_err());
}

4
rust/myfsio-engine/crates/myfsio-storage/src/lib.rs
View File

@@ -1,4 +1,4 @@
pub mod validation;
pub mod traits;
pub mod error;
pub mod fs_backend;
pub mod traits;
pub mod validation;

37
rust/myfsio-engine/crates/myfsio-storage/src/traits.rs
View File

@@ -24,7 +24,11 @@ pub trait StorageEngine: Send + Sync {
metadata: Option<HashMap<String, String>>,
) -> StorageResult<ObjectMeta>;
async fn get_object(&self, bucket: &str, key: &str) -> StorageResult<(ObjectMeta, AsyncReadStream)>;
async fn get_object(
&self,
bucket: &str,
key: &str,
) -> StorageResult<(ObjectMeta, AsyncReadStream)>;
async fn get_object_path(&self, bucket: &str, key: &str) -> StorageResult<PathBuf>;
@@ -53,7 +57,11 @@ pub trait StorageEngine: Send + Sync {
metadata: &HashMap<String, String>,
) -> StorageResult<()>;
async fn list_objects(&self, bucket: &str, params: &ListParams) -> StorageResult<ListObjectsResult>;
async fn list_objects(
&self,
bucket: &str,
params: &ListParams,
) -> StorageResult<ListObjectsResult>;
async fn list_objects_shallow(
&self,
@@ -97,10 +105,8 @@ pub trait StorageEngine: Send + Sync {
async fn list_parts(&self, bucket: &str, upload_id: &str) -> StorageResult<Vec<PartMeta>>;
async fn list_multipart_uploads(
&self,
bucket: &str,
) -> StorageResult<Vec<MultipartUploadInfo>>;
async fn list_multipart_uploads(&self, bucket: &str)
-> StorageResult<Vec<MultipartUploadInfo>>;
async fn get_bucket_config(&self, bucket: &str) -> StorageResult<BucketConfig>;
async fn set_bucket_config(&self, bucket: &str, config: &BucketConfig) -> StorageResult<()>;
@@ -114,22 +120,9 @@ pub trait StorageEngine: Send + Sync {
key: &str,
) -> StorageResult<Vec<VersionInfo>>;
async fn get_object_tags(
&self,
bucket: &str,
key: &str,
) -> StorageResult<Vec<Tag>>;
async fn get_object_tags(&self, bucket: &str, key: &str) -> StorageResult<Vec<Tag>>;
async fn set_object_tags(
&self,
bucket: &str,
key: &str,
tags: &[Tag],
) -> StorageResult<()>;
async fn set_object_tags(&self, bucket: &str, key: &str, tags: &[Tag]) -> StorageResult<()>;
async fn delete_object_tags(
&self,
bucket: &str,
key: &str,
) -> StorageResult<()>;
async fn delete_object_tags(&self, bucket: &str, key: &str) -> StorageResult<()>;
}

11
rust/myfsio-engine/crates/myfsio-storage/src/validation.rs
View File

@@ -3,8 +3,7 @@ use unicode_normalization::UnicodeNormalization;
const WINDOWS_RESERVED: &[&str] = &[
"CON", "PRN", "AUX", "NUL", "COM0", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7",
"COM8", "COM9", "LPT0", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8",
"LPT9",
"COM8", "COM9", "LPT0", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
];
const WINDOWS_ILLEGAL_CHARS: &[char] = &['<', '>', ':', '"', '/', '\\', '|', '?', '*'];
@@ -110,14 +109,10 @@ pub fn validate_bucket_name(bucket_name: &str) -> Option<String> {
let bytes = bucket_name.as_bytes();
if !bytes[0].is_ascii_lowercase() && !bytes[0].is_ascii_digit() {
return Some(
"Bucket name must start and end with a lowercase letter or digit".to_string(),
);
return Some("Bucket name must start and end with a lowercase letter or digit".to_string());
}
if !bytes[len - 1].is_ascii_lowercase() && !bytes[len - 1].is_ascii_digit() {
return Some(
"Bucket name must start and end with a lowercase letter or digit".to_string(),
);
return Some("Bucket name must start and end with a lowercase letter or digit".to_string());
}
for &b in bytes {

4
rust/myfsio-engine/crates/myfsio-xml/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "myfsio-xml"
version = "0.1.0"
edition = "2021"
version.workspace = true
edition.workspace = true
[dependencies]
myfsio-common = { path = "../myfsio-common" }

2
rust/myfsio-engine/crates/myfsio-xml/src/lib.rs
View File

@@ -1,5 +1,5 @@
pub mod response;
pub mod request;
pub mod response;
use quick_xml::Writer;
use std::io::Cursor;

6
rust/myfsio-engine/crates/myfsio-xml/src/request.rs
View File

@@ -49,7 +49,11 @@ pub fn parse_complete_multipart_upload(xml: &str) -> Result<CompleteMultipartUpl
let text = e.unescape().map_err(|e| e.to_string())?.to_string();
match current_tag.as_str() {
"PartNumber" => {
part_number = Some(text.trim().parse().map_err(|e: std::num::ParseIntError| e.to_string())?);
part_number = Some(
text.trim()
.parse()
.map_err(|e: std::num::ParseIntError| e.to_string())?,
);
}
"ETag" => {
etag = Some(text.trim().trim_matches('"').to_string());

213
rust/myfsio-engine/crates/myfsio-xml/src/response.rs
View File

@@ -11,27 +11,47 @@ pub fn format_s3_datetime(dt: &DateTime<Utc>) -> String {
pub fn list_buckets_xml(owner_id: &str, owner_name: &str, buckets: &[BucketMeta]) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
writer
.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))
.unwrap();
let start = BytesStart::new("ListAllMyBucketsResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
writer.write_event(Event::Start(start)).unwrap();
writer.write_event(Event::Start(BytesStart::new("Owner"))).unwrap();
writer
.write_event(Event::Start(BytesStart::new("Owner")))
.unwrap();
write_text_element(&mut writer, "ID", owner_id);
write_text_element(&mut writer, "DisplayName", owner_name);
writer.write_event(Event::End(BytesEnd::new("Owner"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("Owner")))
.unwrap();
writer.write_event(Event::Start(BytesStart::new("Buckets"))).unwrap();
writer
.write_event(Event::Start(BytesStart::new("Buckets")))
.unwrap();
for bucket in buckets {
writer.write_event(Event::Start(BytesStart::new("Bucket"))).unwrap();
writer
.write_event(Event::Start(BytesStart::new("Bucket")))
.unwrap();
write_text_element(&mut writer, "Name", &bucket.name);
write_text_element(&mut writer, "CreationDate", &format_s3_datetime(&bucket.creation_date));
writer.write_event(Event::End(BytesEnd::new("Bucket"))).unwrap();
write_text_element(
&mut writer,
"CreationDate",
&format_s3_datetime(&bucket.creation_date),
);
writer
.write_event(Event::End(BytesEnd::new("Bucket")))
.unwrap();
}
writer.write_event(Event::End(BytesEnd::new("Buckets"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("Buckets")))
.unwrap();
writer.write_event(Event::End(BytesEnd::new("ListAllMyBucketsResult"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("ListAllMyBucketsResult")))
.unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
@@ -50,7 +70,9 @@ pub fn list_objects_v2_xml(
) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
writer
.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))
.unwrap();
let start = BytesStart::new("ListBucketResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
@@ -73,24 +95,42 @@ pub fn list_objects_v2_xml(
}
for obj in objects {
writer.write_event(Event::Start(BytesStart::new("Contents"))).unwrap();
writer
.write_event(Event::Start(BytesStart::new("Contents")))
.unwrap();
write_text_element(&mut writer, "Key", &obj.key);
write_text_element(&mut writer, "LastModified", &format_s3_datetime(&obj.last_modified));
write_text_element(
&mut writer,
"LastModified",
&format_s3_datetime(&obj.last_modified),
);
if let Some(ref etag) = obj.etag {
write_text_element(&mut writer, "ETag", &format!("\"{}\"", etag));
}
write_text_element(&mut writer, "Size", &obj.size.to_string());
write_text_element(&mut writer, "StorageClass", obj.storage_class.as_deref().unwrap_or("STANDARD"));
writer.write_event(Event::End(BytesEnd::new("Contents"))).unwrap();
write_text_element(
&mut writer,
"StorageClass",
obj.storage_class.as_deref().unwrap_or("STANDARD"),
);
writer
.write_event(Event::End(BytesEnd::new("Contents")))
.unwrap();
}
for prefix in common_prefixes {
writer.write_event(Event::Start(BytesStart::new("CommonPrefixes"))).unwrap();
writer
.write_event(Event::Start(BytesStart::new("CommonPrefixes")))
.unwrap();
write_text_element(&mut writer, "Prefix", prefix);
writer.write_event(Event::End(BytesEnd::new("CommonPrefixes"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("CommonPrefixes")))
.unwrap();
}
writer.write_event(Event::End(BytesEnd::new("ListBucketResult"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("ListBucketResult")))
.unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
@@ -138,7 +178,11 @@ pub fn list_objects_v1_xml(
.write_event(Event::Start(BytesStart::new("Contents")))
.unwrap();
write_text_element(&mut writer, "Key", &obj.key);
write_text_element(&mut writer, "LastModified", &format_s3_datetime(&obj.last_modified));
write_text_element(
&mut writer,
"LastModified",
&format_s3_datetime(&obj.last_modified),
);
if let Some(ref etag) = obj.etag {
write_text_element(&mut writer, "ETag", &format!("\"{}\"", etag));
}
@@ -166,14 +210,20 @@ pub fn list_objects_v1_xml(
}
fn write_text_element(writer: &mut Writer<Cursor<Vec<u8>>>, tag: &str, text: &str) {
writer.write_event(Event::Start(BytesStart::new(tag))).unwrap();
writer.write_event(Event::Text(BytesText::new(text))).unwrap();
writer
.write_event(Event::Start(BytesStart::new(tag)))
.unwrap();
writer
.write_event(Event::Text(BytesText::new(text)))
.unwrap();
writer.write_event(Event::End(BytesEnd::new(tag))).unwrap();
}
pub fn initiate_multipart_upload_xml(bucket: &str, key: &str, upload_id: &str) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
writer
.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))
.unwrap();
let start = BytesStart::new("InitiateMultipartUploadResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
@@ -181,7 +231,9 @@ pub fn initiate_multipart_upload_xml(bucket: &str, key: &str, upload_id: &str) -
write_text_element(&mut writer, "Bucket", bucket);
write_text_element(&mut writer, "Key", key);
write_text_element(&mut writer, "UploadId", upload_id);
writer.write_event(Event::End(BytesEnd::new("InitiateMultipartUploadResult"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("InitiateMultipartUploadResult")))
.unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
@@ -193,7 +245,9 @@ pub fn complete_multipart_upload_xml(
location: &str,
) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
writer
.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))
.unwrap();
let start = BytesStart::new("CompleteMultipartUploadResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
@@ -202,28 +256,36 @@ pub fn complete_multipart_upload_xml(
write_text_element(&mut writer, "Bucket", bucket);
write_text_element(&mut writer, "Key", key);
write_text_element(&mut writer, "ETag", &format!("\"{}\"", etag));
writer.write_event(Event::End(BytesEnd::new("CompleteMultipartUploadResult"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("CompleteMultipartUploadResult")))
.unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
pub fn copy_part_result_xml(etag: &str, last_modified: &str) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
writer
.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))
.unwrap();
let start = BytesStart::new("CopyPartResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
writer.write_event(Event::Start(start)).unwrap();
write_text_element(&mut writer, "LastModified", last_modified);
write_text_element(&mut writer, "ETag", &format!("\"{}\"", etag));
writer.write_event(Event::End(BytesEnd::new("CopyPartResult"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("CopyPartResult")))
.unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
pub fn post_object_result_xml(location: &str, bucket: &str, key: &str, etag: &str) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
writer
.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))
.unwrap();
let start = BytesStart::new("PostResponse")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
@@ -232,21 +294,27 @@ pub fn post_object_result_xml(location: &str, bucket: &str, key: &str, etag: &st
write_text_element(&mut writer, "Bucket", bucket);
write_text_element(&mut writer, "Key", key);
write_text_element(&mut writer, "ETag", &format!("\"{}\"", etag));
writer.write_event(Event::End(BytesEnd::new("PostResponse"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("PostResponse")))
.unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
pub fn copy_object_result_xml(etag: &str, last_modified: &str) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
writer
.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))
.unwrap();
let start = BytesStart::new("CopyObjectResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
writer.write_event(Event::Start(start)).unwrap();
write_text_element(&mut writer, "ETag", &format!("\"{}\"", etag));
write_text_element(&mut writer, "LastModified", last_modified);
writer.write_event(Event::End(BytesEnd::new("CopyObjectResult"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("CopyObjectResult")))
.unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
@@ -257,7 +325,9 @@ pub fn delete_result_xml(
quiet: bool,
) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
writer
.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))
.unwrap();
let start = BytesStart::new("DeleteResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
@@ -265,24 +335,34 @@ pub fn delete_result_xml(
if !quiet {
for (key, version_id) in deleted {
writer.write_event(Event::Start(BytesStart::new("Deleted"))).unwrap();
writer
.write_event(Event::Start(BytesStart::new("Deleted")))
.unwrap();
write_text_element(&mut writer, "Key", key);
if let Some(vid) = version_id {
write_text_element(&mut writer, "VersionId", vid);
}
writer.write_event(Event::End(BytesEnd::new("Deleted"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("Deleted")))
.unwrap();
}
}
for (key, code, message) in errors {
writer.write_event(Event::Start(BytesStart::new("Error"))).unwrap();
writer
.write_event(Event::Start(BytesStart::new("Error")))
.unwrap();
write_text_element(&mut writer, "Key", key);
write_text_element(&mut writer, "Code", code);
write_text_element(&mut writer, "Message", message);
writer.write_event(Event::End(BytesEnd::new("Error"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("Error")))
.unwrap();
}
writer.write_event(Event::End(BytesEnd::new("DeleteResult"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("DeleteResult")))
.unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
@@ -292,7 +372,9 @@ pub fn list_multipart_uploads_xml(
uploads: &[myfsio_common::types::MultipartUploadInfo],
) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
writer
.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))
.unwrap();
let start = BytesStart::new("ListMultipartUploadsResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
@@ -300,14 +382,24 @@ pub fn list_multipart_uploads_xml(
write_text_element(&mut writer, "Bucket", bucket);
for upload in uploads {
writer.write_event(Event::Start(BytesStart::new("Upload"))).unwrap();
writer
.write_event(Event::Start(BytesStart::new("Upload")))
.unwrap();
write_text_element(&mut writer, "Key", &upload.key);
write_text_element(&mut writer, "UploadId", &upload.upload_id);
write_text_element(&mut writer, "Initiated", &format_s3_datetime(&upload.initiated));
writer.write_event(Event::End(BytesEnd::new("Upload"))).unwrap();
write_text_element(
&mut writer,
"Initiated",
&format_s3_datetime(&upload.initiated),
);
writer
.write_event(Event::End(BytesEnd::new("Upload")))
.unwrap();
}
writer.write_event(Event::End(BytesEnd::new("ListMultipartUploadsResult"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("ListMultipartUploadsResult")))
.unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
@@ -319,7 +411,9 @@ pub fn list_parts_xml(
parts: &[myfsio_common::types::PartMeta],
) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
writer
.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))
.unwrap();
let start = BytesStart::new("ListPartsResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
@@ -329,17 +423,23 @@ pub fn list_parts_xml(
write_text_element(&mut writer, "UploadId", upload_id);
for part in parts {
writer.write_event(Event::Start(BytesStart::new("Part"))).unwrap();
writer
.write_event(Event::Start(BytesStart::new("Part")))
.unwrap();
write_text_element(&mut writer, "PartNumber", &part.part_number.to_string());
write_text_element(&mut writer, "ETag", &format!("\"{}\"", part.etag));
write_text_element(&mut writer, "Size", &part.size.to_string());
if let Some(ref lm) = part.last_modified {
write_text_element(&mut writer, "LastModified", &format_s3_datetime(lm));
}
writer.write_event(Event::End(BytesEnd::new("Part"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("Part")))
.unwrap();
}
writer.write_event(Event::End(BytesEnd::new("ListPartsResult"))).unwrap();
writer
.write_event(Event::End(BytesEnd::new("ListPartsResult")))
.unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
@@ -365,7 +465,16 @@ mod tests {
fn test_list_objects_v2_xml() {
let objects = vec![ObjectMeta::new("file.txt".to_string(), 1024, Utc::now())];
let xml = list_objects_v2_xml(
"my-bucket", "", "/", 1000, &objects, &[], false, None, None, 1,
"my-bucket",
"",
"/",
1000,
&objects,
&[],
false,
None,
None,
1,
);
assert!(xml.contains("<Key>file.txt</Key>"));
assert!(xml.contains("<Size>1024</Size>"));
@@ -375,17 +484,7 @@ mod tests {
#[test]
fn test_list_objects_v1_xml() {
let objects = vec![ObjectMeta::new("file.txt".to_string(), 1024, Utc::now())];
let xml = list_objects_v1_xml(
"my-bucket",
"",
"",
"/",
1000,
&objects,
&[],
false,
None,
);
let xml = list_objects_v1_xml("my-bucket", "", "", "/", 1000, &objects, &[], false, None);
assert!(xml.contains("<Key>file.txt</Key>"));
assert!(xml.contains("<Size>1024</Size>"));
assert!(xml.contains("<Marker></Marker>"));