Compare commits
62 Commits
217af6d1c6
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| d434c95e7f | |||
| 37dcda282f | |||
| 46267b4f78 | |||
| d57ea8378a | |||
| 4d923df16c | |||
| b4e2e15936 | |||
| 05a30d2227 | |||
| 02fa9d612c | |||
| 6c5ccee8cb | |||
| 0a60ea4348 | |||
| 069049b146 | |||
| 6ba948bcc0 | |||
| b5facd8d37 | |||
| 1c9ebdeab7 | |||
| 777d862a02 | |||
| 660c328a84 | |||
| 3a590e6639 | |||
| dd1e6d0409 | |||
| 7e32ac2a46 | |||
| 37541ffba1 | |||
| 5aba9ac9e9 | |||
| 4f05192548 | |||
| 1ea6dfae07 | |||
| f2df64479c | |||
| bd405cc2fe | |||
| 7ef3820f6e | |||
| e1fb225034 | |||
| 2767e7e79d | |||
| ae11c654f9 | |||
| f0c95ac0a9 | |||
| 8ff4797041 | |||
| 50fb5aa387 | |||
| cc161bf362 | |||
| 2a0e77a754 | |||
| eb0e435a5a | |||
| 7633007a08 | |||
| de0d869c9f | |||
| fdd068feee | |||
| 66b7677d2c | |||
| 4d90ead816 | |||
| b37a51ed1d | |||
| 0462a7b62e | |||
| 52660570c1 | |||
| 35f61313e0 | |||
| c470cfb576 | |||
| d96955deee | |||
| 85181f0be6 | |||
| d5ca7a8be1 | |||
| 476dc79e42 | |||
| bb6590fc5e | |||
| 899db3421b | |||
| caf01d6ada | |||
| bb366cb4cd | |||
| a2745ff2ee | |||
| 28cb656d94 | |||
| 3c44152fc6 | |||
| 397515edce | |||
| 980fced7e4 | |||
| bae5009ec4 | |||
| 233780617f | |||
| fd8fb21517 | |||
| c6cbe822e1 |
117
Cargo.lock
generated
117
Cargo.lock
generated
@@ -1460,6 +1460,27 @@ dependencies = [
|
|||||||
"crypto-common 0.2.1",
|
"crypto-common 0.2.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "dirs"
|
||||||
|
version = "6.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e"
|
||||||
|
dependencies = [
|
||||||
|
"dirs-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "dirs-sys"
|
||||||
|
version = "0.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"option-ext",
|
||||||
|
"redox_users",
|
||||||
|
"windows-sys 0.60.2",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "displaydoc"
|
name = "displaydoc"
|
||||||
version = "0.2.5"
|
version = "0.2.5"
|
||||||
@@ -2639,7 +2660,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "myfsio-auth"
|
name = "myfsio-auth"
|
||||||
version = "0.4.3"
|
version = "0.5.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes",
|
"aes",
|
||||||
"base64",
|
"base64",
|
||||||
@@ -2664,18 +2685,20 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "myfsio-common"
|
name = "myfsio-common"
|
||||||
version = "0.4.3"
|
version = "0.5.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"base64",
|
||||||
"chrono",
|
"chrono",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"sha2 0.10.9",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"uuid",
|
"uuid",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "myfsio-crypto"
|
name = "myfsio-crypto"
|
||||||
version = "0.4.3"
|
version = "0.5.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
"base64",
|
"base64",
|
||||||
@@ -2696,7 +2719,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "myfsio-server"
|
name = "myfsio-server"
|
||||||
version = "0.4.3"
|
version = "0.5.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -2708,12 +2731,15 @@ dependencies = [
|
|||||||
"base64",
|
"base64",
|
||||||
"bytes",
|
"bytes",
|
||||||
"chrono",
|
"chrono",
|
||||||
|
"chrono-tz",
|
||||||
"clap",
|
"clap",
|
||||||
"cookie",
|
"cookie",
|
||||||
"crc32fast",
|
"crc32fast",
|
||||||
"dotenvy",
|
"dotenvy",
|
||||||
"duckdb",
|
"duckdb",
|
||||||
"futures",
|
"futures",
|
||||||
|
"hex",
|
||||||
|
"http-body 1.0.1",
|
||||||
"http-body-util",
|
"http-body-util",
|
||||||
"hyper 1.9.0",
|
"hyper 1.9.0",
|
||||||
"md-5 0.10.6",
|
"md-5 0.10.6",
|
||||||
@@ -2731,6 +2757,7 @@ dependencies = [
|
|||||||
"regex",
|
"regex",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"roxmltree",
|
"roxmltree",
|
||||||
|
"rust-embed",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_urlencoded",
|
"serde_urlencoded",
|
||||||
@@ -2740,6 +2767,7 @@ dependencies = [
|
|||||||
"tempfile",
|
"tempfile",
|
||||||
"tera",
|
"tera",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tokio-stream",
|
||||||
"tokio-util",
|
"tokio-util",
|
||||||
"tower",
|
"tower",
|
||||||
"tower-http",
|
"tower-http",
|
||||||
@@ -2750,7 +2778,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "myfsio-storage"
|
name = "myfsio-storage"
|
||||||
version = "0.4.3"
|
version = "0.5.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"chrono",
|
"chrono",
|
||||||
"dashmap",
|
"dashmap",
|
||||||
@@ -2766,6 +2794,7 @@ dependencies = [
|
|||||||
"tempfile",
|
"tempfile",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tokio-util",
|
||||||
"tracing",
|
"tracing",
|
||||||
"unicode-normalization",
|
"unicode-normalization",
|
||||||
"uuid",
|
"uuid",
|
||||||
@@ -2773,12 +2802,15 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "myfsio-xml"
|
name = "myfsio-xml"
|
||||||
version = "0.4.3"
|
version = "0.5.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"base64",
|
||||||
"chrono",
|
"chrono",
|
||||||
"myfsio-common",
|
"myfsio-common",
|
||||||
|
"percent-encoding",
|
||||||
"quick-xml",
|
"quick-xml",
|
||||||
"serde",
|
"serde",
|
||||||
|
"sha2 0.10.9",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -2867,6 +2899,12 @@ version = "0.2.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
|
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "option-ext"
|
||||||
|
version = "0.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "outref"
|
name = "outref"
|
||||||
version = "0.5.2"
|
version = "0.5.2"
|
||||||
@@ -3320,6 +3358,17 @@ dependencies = [
|
|||||||
"bitflags",
|
"bitflags",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "redox_users"
|
||||||
|
version = "0.5.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac"
|
||||||
|
dependencies = [
|
||||||
|
"getrandom 0.2.17",
|
||||||
|
"libredox",
|
||||||
|
"thiserror",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "regex"
|
name = "regex"
|
||||||
version = "1.12.3"
|
version = "1.12.3"
|
||||||
@@ -3466,6 +3515,42 @@ version = "0.20.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97"
|
checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rust-embed"
|
||||||
|
version = "8.11.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27"
|
||||||
|
dependencies = [
|
||||||
|
"rust-embed-impl",
|
||||||
|
"rust-embed-utils",
|
||||||
|
"walkdir",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rust-embed-impl"
|
||||||
|
version = "8.11.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"rust-embed-utils",
|
||||||
|
"shellexpand",
|
||||||
|
"syn 2.0.117",
|
||||||
|
"walkdir",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rust-embed-utils"
|
||||||
|
version = "8.11.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1"
|
||||||
|
dependencies = [
|
||||||
|
"globset",
|
||||||
|
"sha2 0.10.9",
|
||||||
|
"walkdir",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rust_decimal"
|
name = "rust_decimal"
|
||||||
version = "1.41.0"
|
version = "1.41.0"
|
||||||
@@ -3796,6 +3881,15 @@ dependencies = [
|
|||||||
"lazy_static",
|
"lazy_static",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "shellexpand"
|
||||||
|
version = "3.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8"
|
||||||
|
dependencies = [
|
||||||
|
"dirs",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "shlex"
|
name = "shlex"
|
||||||
version = "1.3.0"
|
version = "1.3.0"
|
||||||
@@ -4193,6 +4287,17 @@ dependencies = [
|
|||||||
"tokio",
|
"tokio",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tokio-stream"
|
||||||
|
version = "0.1.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"
|
||||||
|
dependencies = [
|
||||||
|
"futures-core",
|
||||||
|
"pin-project-lite",
|
||||||
|
"tokio",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokio-util"
|
name = "tokio-util"
|
||||||
version = "0.7.18"
|
version = "0.7.18"
|
||||||
|
|||||||
@@ -10,14 +10,14 @@ members = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
version = "0.4.3"
|
version = "0.5.1"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
tokio = { version = "1", features = ["full"] }
|
tokio = { version = "1", features = ["full"] }
|
||||||
axum = { version = "0.8" }
|
axum = { version = "0.8" }
|
||||||
tower = { version = "0.5" }
|
tower = { version = "0.5" }
|
||||||
tower-http = { version = "0.6", features = ["cors", "trace", "fs", "compression-gzip"] }
|
tower-http = { version = "0.6", features = ["cors", "trace", "fs", "compression-gzip", "timeout", "set-header"] }
|
||||||
hyper = { version = "1" }
|
hyper = { version = "1" }
|
||||||
bytes = "1"
|
bytes = "1"
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
@@ -41,8 +41,10 @@ tracing = "0.1"
|
|||||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
thiserror = "2"
|
thiserror = "2"
|
||||||
chrono = { version = "0.4", features = ["serde"] }
|
chrono = { version = "0.4", features = ["serde"] }
|
||||||
|
chrono-tz = "0.9"
|
||||||
base64 = "0.22"
|
base64 = "0.22"
|
||||||
tokio-util = { version = "0.7", features = ["io"] }
|
tokio-util = { version = "0.7", features = ["io", "io-util"] }
|
||||||
|
tokio-stream = "0.1"
|
||||||
futures = "0.3"
|
futures = "0.3"
|
||||||
dashmap = "6"
|
dashmap = "6"
|
||||||
crc32fast = "1"
|
crc32fast = "1"
|
||||||
|
|||||||
@@ -77,17 +77,66 @@ impl RawIamUser {
|
|||||||
let user_id = self.user_id.unwrap_or_else(|| {
|
let user_id = self.user_id.unwrap_or_else(|| {
|
||||||
format!("u-{}", display_name.to_ascii_lowercase().replace(' ', "-"))
|
format!("u-{}", display_name.to_ascii_lowercase().replace(' ', "-"))
|
||||||
});
|
});
|
||||||
|
let policies = self
|
||||||
|
.policies
|
||||||
|
.into_iter()
|
||||||
|
.map(normalize_legacy_full_access)
|
||||||
|
.collect();
|
||||||
IamUser {
|
IamUser {
|
||||||
user_id,
|
user_id,
|
||||||
display_name,
|
display_name,
|
||||||
enabled: self.enabled,
|
enabled: self.enabled,
|
||||||
expires_at: self.expires_at,
|
expires_at: self.expires_at,
|
||||||
access_keys,
|
access_keys,
|
||||||
policies: self.policies,
|
policies,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const LEGACY_FULL_ACCESS_ACTIONS: &[&str] = &[
|
||||||
|
"list",
|
||||||
|
"read",
|
||||||
|
"write",
|
||||||
|
"delete",
|
||||||
|
"share",
|
||||||
|
"policy",
|
||||||
|
"create_bucket",
|
||||||
|
"delete_bucket",
|
||||||
|
"replication",
|
||||||
|
"lifecycle",
|
||||||
|
"cors",
|
||||||
|
"versioning",
|
||||||
|
"tagging",
|
||||||
|
"encryption",
|
||||||
|
"quota",
|
||||||
|
"object_lock",
|
||||||
|
"notification",
|
||||||
|
"logging",
|
||||||
|
"website",
|
||||||
|
];
|
||||||
|
|
||||||
|
fn normalize_legacy_full_access(policy: IamPolicy) -> IamPolicy {
|
||||||
|
if policy.bucket != "*"
|
||||||
|
|| policy.prefix != "*"
|
||||||
|
|| policy.actions.iter().any(|a| a == "*")
|
||||||
|
{
|
||||||
|
return policy;
|
||||||
|
}
|
||||||
|
if !policy.actions.iter().any(|a| a == "iam:*") {
|
||||||
|
return policy;
|
||||||
|
}
|
||||||
|
for required in LEGACY_FULL_ACCESS_ACTIONS {
|
||||||
|
if !policy.actions.iter().any(|a| a == *required) {
|
||||||
|
return policy;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
IamPolicy {
|
||||||
|
bucket: policy.bucket,
|
||||||
|
prefix: policy.prefix,
|
||||||
|
actions: vec!["*".to_string()],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn default_enabled() -> bool {
|
fn default_enabled() -> bool {
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
mod fernet;
|
pub mod fernet;
|
||||||
pub mod iam;
|
pub mod iam;
|
||||||
pub mod principal;
|
pub mod principal;
|
||||||
pub mod sigv4;
|
pub mod sigv4;
|
||||||
|
|||||||
@@ -31,13 +31,13 @@ fn hmac_sha256(key: &[u8], msg: &[u8]) -> Vec<u8> {
|
|||||||
mac.finalize().into_bytes().to_vec()
|
mac.finalize().into_bytes().to_vec()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sha256_hex(data: &[u8]) -> String {
|
pub fn sha256_hex(data: &[u8]) -> String {
|
||||||
let mut hasher = Sha256::new();
|
let mut hasher = Sha256::new();
|
||||||
hasher.update(data);
|
hasher.update(data);
|
||||||
hex::encode(hasher.finalize())
|
hex::encode(hasher.finalize())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn aws_uri_encode(input: &str) -> String {
|
pub fn aws_uri_encode(input: &str) -> String {
|
||||||
percent_encode(input.as_bytes(), AWS_ENCODE_SET).to_string()
|
percent_encode(input.as_bytes(), AWS_ENCODE_SET).to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,3 +9,5 @@ serde = { workspace = true }
|
|||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
chrono = { workspace = true }
|
chrono = { workspace = true }
|
||||||
uuid = { workspace = true }
|
uuid = { workspace = true }
|
||||||
|
sha2 = { workspace = true }
|
||||||
|
base64 = { workspace = true }
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ pub const STATS_FILE: &str = "stats.json";
|
|||||||
pub const ETAG_INDEX_FILE: &str = "etag_index.json";
|
pub const ETAG_INDEX_FILE: &str = "etag_index.json";
|
||||||
pub const INDEX_FILE: &str = "_index.json";
|
pub const INDEX_FILE: &str = "_index.json";
|
||||||
pub const MANIFEST_FILE: &str = "manifest.json";
|
pub const MANIFEST_FILE: &str = "manifest.json";
|
||||||
|
pub const DIR_MARKER_FILE: &str = ".__myfsio_dirobj__";
|
||||||
|
|
||||||
pub const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"];
|
pub const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"];
|
||||||
|
|
||||||
|
|||||||
@@ -5,13 +5,17 @@ pub enum S3ErrorCode {
|
|||||||
AccessDenied,
|
AccessDenied,
|
||||||
BadDigest,
|
BadDigest,
|
||||||
BucketAlreadyExists,
|
BucketAlreadyExists,
|
||||||
|
BucketAlreadyOwnedByYou,
|
||||||
BucketNotEmpty,
|
BucketNotEmpty,
|
||||||
EntityTooLarge,
|
EntityTooLarge,
|
||||||
|
EntityTooSmall,
|
||||||
InternalError,
|
InternalError,
|
||||||
InvalidAccessKeyId,
|
InvalidAccessKeyId,
|
||||||
InvalidArgument,
|
InvalidArgument,
|
||||||
InvalidBucketName,
|
InvalidBucketName,
|
||||||
InvalidKey,
|
InvalidKey,
|
||||||
|
InvalidPart,
|
||||||
|
InvalidPartOrder,
|
||||||
InvalidPolicyDocument,
|
InvalidPolicyDocument,
|
||||||
InvalidRange,
|
InvalidRange,
|
||||||
InvalidRequest,
|
InvalidRequest,
|
||||||
@@ -19,13 +23,18 @@ pub enum S3ErrorCode {
|
|||||||
MalformedXML,
|
MalformedXML,
|
||||||
MethodNotAllowed,
|
MethodNotAllowed,
|
||||||
NoSuchBucket,
|
NoSuchBucket,
|
||||||
|
NoSuchBucketPolicy,
|
||||||
NoSuchKey,
|
NoSuchKey,
|
||||||
|
NoSuchLifecycleConfiguration,
|
||||||
NoSuchUpload,
|
NoSuchUpload,
|
||||||
NoSuchVersion,
|
NoSuchVersion,
|
||||||
NoSuchTagSet,
|
NoSuchTagSet,
|
||||||
|
ObjectCorrupted,
|
||||||
PreconditionFailed,
|
PreconditionFailed,
|
||||||
NotModified,
|
NotModified,
|
||||||
QuotaExceeded,
|
QuotaExceeded,
|
||||||
|
RequestTimeTooSkewed,
|
||||||
|
ServerSideEncryptionConfigurationNotFoundError,
|
||||||
SignatureDoesNotMatch,
|
SignatureDoesNotMatch,
|
||||||
SlowDown,
|
SlowDown,
|
||||||
}
|
}
|
||||||
@@ -36,13 +45,17 @@ impl S3ErrorCode {
|
|||||||
Self::AccessDenied => 403,
|
Self::AccessDenied => 403,
|
||||||
Self::BadDigest => 400,
|
Self::BadDigest => 400,
|
||||||
Self::BucketAlreadyExists => 409,
|
Self::BucketAlreadyExists => 409,
|
||||||
|
Self::BucketAlreadyOwnedByYou => 409,
|
||||||
Self::BucketNotEmpty => 409,
|
Self::BucketNotEmpty => 409,
|
||||||
Self::EntityTooLarge => 413,
|
Self::EntityTooLarge => 413,
|
||||||
|
Self::EntityTooSmall => 400,
|
||||||
Self::InternalError => 500,
|
Self::InternalError => 500,
|
||||||
Self::InvalidAccessKeyId => 403,
|
Self::InvalidAccessKeyId => 403,
|
||||||
Self::InvalidArgument => 400,
|
Self::InvalidArgument => 400,
|
||||||
Self::InvalidBucketName => 400,
|
Self::InvalidBucketName => 400,
|
||||||
Self::InvalidKey => 400,
|
Self::InvalidKey => 400,
|
||||||
|
Self::InvalidPart => 400,
|
||||||
|
Self::InvalidPartOrder => 400,
|
||||||
Self::InvalidPolicyDocument => 400,
|
Self::InvalidPolicyDocument => 400,
|
||||||
Self::InvalidRange => 416,
|
Self::InvalidRange => 416,
|
||||||
Self::InvalidRequest => 400,
|
Self::InvalidRequest => 400,
|
||||||
@@ -50,15 +63,20 @@ impl S3ErrorCode {
|
|||||||
Self::MalformedXML => 400,
|
Self::MalformedXML => 400,
|
||||||
Self::MethodNotAllowed => 405,
|
Self::MethodNotAllowed => 405,
|
||||||
Self::NoSuchBucket => 404,
|
Self::NoSuchBucket => 404,
|
||||||
|
Self::NoSuchBucketPolicy => 404,
|
||||||
Self::NoSuchKey => 404,
|
Self::NoSuchKey => 404,
|
||||||
|
Self::NoSuchLifecycleConfiguration => 404,
|
||||||
Self::NoSuchUpload => 404,
|
Self::NoSuchUpload => 404,
|
||||||
Self::NoSuchVersion => 404,
|
Self::NoSuchVersion => 404,
|
||||||
Self::NoSuchTagSet => 404,
|
Self::NoSuchTagSet => 404,
|
||||||
|
Self::ObjectCorrupted => 422,
|
||||||
Self::PreconditionFailed => 412,
|
Self::PreconditionFailed => 412,
|
||||||
Self::NotModified => 304,
|
Self::NotModified => 304,
|
||||||
Self::QuotaExceeded => 403,
|
Self::QuotaExceeded => 403,
|
||||||
|
Self::RequestTimeTooSkewed => 403,
|
||||||
|
Self::ServerSideEncryptionConfigurationNotFoundError => 404,
|
||||||
Self::SignatureDoesNotMatch => 403,
|
Self::SignatureDoesNotMatch => 403,
|
||||||
Self::SlowDown => 429,
|
Self::SlowDown => 503,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -67,13 +85,17 @@ impl S3ErrorCode {
|
|||||||
Self::AccessDenied => "AccessDenied",
|
Self::AccessDenied => "AccessDenied",
|
||||||
Self::BadDigest => "BadDigest",
|
Self::BadDigest => "BadDigest",
|
||||||
Self::BucketAlreadyExists => "BucketAlreadyExists",
|
Self::BucketAlreadyExists => "BucketAlreadyExists",
|
||||||
|
Self::BucketAlreadyOwnedByYou => "BucketAlreadyOwnedByYou",
|
||||||
Self::BucketNotEmpty => "BucketNotEmpty",
|
Self::BucketNotEmpty => "BucketNotEmpty",
|
||||||
Self::EntityTooLarge => "EntityTooLarge",
|
Self::EntityTooLarge => "EntityTooLarge",
|
||||||
|
Self::EntityTooSmall => "EntityTooSmall",
|
||||||
Self::InternalError => "InternalError",
|
Self::InternalError => "InternalError",
|
||||||
Self::InvalidAccessKeyId => "InvalidAccessKeyId",
|
Self::InvalidAccessKeyId => "InvalidAccessKeyId",
|
||||||
Self::InvalidArgument => "InvalidArgument",
|
Self::InvalidArgument => "InvalidArgument",
|
||||||
Self::InvalidBucketName => "InvalidBucketName",
|
Self::InvalidBucketName => "InvalidBucketName",
|
||||||
Self::InvalidKey => "InvalidKey",
|
Self::InvalidKey => "InvalidKey",
|
||||||
|
Self::InvalidPart => "InvalidPart",
|
||||||
|
Self::InvalidPartOrder => "InvalidPartOrder",
|
||||||
Self::InvalidPolicyDocument => "InvalidPolicyDocument",
|
Self::InvalidPolicyDocument => "InvalidPolicyDocument",
|
||||||
Self::InvalidRange => "InvalidRange",
|
Self::InvalidRange => "InvalidRange",
|
||||||
Self::InvalidRequest => "InvalidRequest",
|
Self::InvalidRequest => "InvalidRequest",
|
||||||
@@ -81,13 +103,20 @@ impl S3ErrorCode {
|
|||||||
Self::MalformedXML => "MalformedXML",
|
Self::MalformedXML => "MalformedXML",
|
||||||
Self::MethodNotAllowed => "MethodNotAllowed",
|
Self::MethodNotAllowed => "MethodNotAllowed",
|
||||||
Self::NoSuchBucket => "NoSuchBucket",
|
Self::NoSuchBucket => "NoSuchBucket",
|
||||||
|
Self::NoSuchBucketPolicy => "NoSuchBucketPolicy",
|
||||||
Self::NoSuchKey => "NoSuchKey",
|
Self::NoSuchKey => "NoSuchKey",
|
||||||
|
Self::NoSuchLifecycleConfiguration => "NoSuchLifecycleConfiguration",
|
||||||
Self::NoSuchUpload => "NoSuchUpload",
|
Self::NoSuchUpload => "NoSuchUpload",
|
||||||
Self::NoSuchVersion => "NoSuchVersion",
|
Self::NoSuchVersion => "NoSuchVersion",
|
||||||
Self::NoSuchTagSet => "NoSuchTagSet",
|
Self::NoSuchTagSet => "NoSuchTagSet",
|
||||||
|
Self::ObjectCorrupted => "ObjectCorrupted",
|
||||||
Self::PreconditionFailed => "PreconditionFailed",
|
Self::PreconditionFailed => "PreconditionFailed",
|
||||||
Self::NotModified => "NotModified",
|
Self::NotModified => "NotModified",
|
||||||
Self::QuotaExceeded => "QuotaExceeded",
|
Self::QuotaExceeded => "QuotaExceeded",
|
||||||
|
Self::RequestTimeTooSkewed => "RequestTimeTooSkewed",
|
||||||
|
Self::ServerSideEncryptionConfigurationNotFoundError => {
|
||||||
|
"ServerSideEncryptionConfigurationNotFoundError"
|
||||||
|
}
|
||||||
Self::SignatureDoesNotMatch => "SignatureDoesNotMatch",
|
Self::SignatureDoesNotMatch => "SignatureDoesNotMatch",
|
||||||
Self::SlowDown => "SlowDown",
|
Self::SlowDown => "SlowDown",
|
||||||
}
|
}
|
||||||
@@ -98,13 +127,17 @@ impl S3ErrorCode {
|
|||||||
Self::AccessDenied => "Access Denied",
|
Self::AccessDenied => "Access Denied",
|
||||||
Self::BadDigest => "The Content-MD5 or checksum value you specified did not match what we received",
|
Self::BadDigest => "The Content-MD5 or checksum value you specified did not match what we received",
|
||||||
Self::BucketAlreadyExists => "The requested bucket name is not available",
|
Self::BucketAlreadyExists => "The requested bucket name is not available",
|
||||||
|
Self::BucketAlreadyOwnedByYou => "Your previous request to create the named bucket succeeded and you already own it",
|
||||||
Self::BucketNotEmpty => "The bucket you tried to delete is not empty",
|
Self::BucketNotEmpty => "The bucket you tried to delete is not empty",
|
||||||
Self::EntityTooLarge => "Your proposed upload exceeds the maximum allowed size",
|
Self::EntityTooLarge => "Your proposed upload exceeds the maximum allowed size",
|
||||||
|
Self::EntityTooSmall => "Your proposed upload is smaller than the minimum allowed object size",
|
||||||
Self::InternalError => "We encountered an internal error. Please try again.",
|
Self::InternalError => "We encountered an internal error. Please try again.",
|
||||||
Self::InvalidAccessKeyId => "The access key ID you provided does not exist",
|
Self::InvalidAccessKeyId => "The access key ID you provided does not exist",
|
||||||
Self::InvalidArgument => "Invalid argument",
|
Self::InvalidArgument => "Invalid argument",
|
||||||
Self::InvalidBucketName => "The specified bucket is not valid",
|
Self::InvalidBucketName => "The specified bucket is not valid",
|
||||||
Self::InvalidKey => "The specified key is not valid",
|
Self::InvalidKey => "The specified key is not valid",
|
||||||
|
Self::InvalidPart => "One or more of the specified parts could not be found",
|
||||||
|
Self::InvalidPartOrder => "The list of parts was not in ascending order",
|
||||||
Self::InvalidPolicyDocument => "The content of the form does not meet the conditions specified in the policy document",
|
Self::InvalidPolicyDocument => "The content of the form does not meet the conditions specified in the policy document",
|
||||||
Self::InvalidRange => "The requested range is not satisfiable",
|
Self::InvalidRange => "The requested range is not satisfiable",
|
||||||
Self::InvalidRequest => "Invalid request",
|
Self::InvalidRequest => "Invalid request",
|
||||||
@@ -112,13 +145,18 @@ impl S3ErrorCode {
|
|||||||
Self::MalformedXML => "The XML you provided was not well-formed",
|
Self::MalformedXML => "The XML you provided was not well-formed",
|
||||||
Self::MethodNotAllowed => "The specified method is not allowed against this resource",
|
Self::MethodNotAllowed => "The specified method is not allowed against this resource",
|
||||||
Self::NoSuchBucket => "The specified bucket does not exist",
|
Self::NoSuchBucket => "The specified bucket does not exist",
|
||||||
|
Self::NoSuchBucketPolicy => "The bucket policy does not exist",
|
||||||
Self::NoSuchKey => "The specified key does not exist",
|
Self::NoSuchKey => "The specified key does not exist",
|
||||||
|
Self::NoSuchLifecycleConfiguration => "The lifecycle configuration does not exist",
|
||||||
Self::NoSuchUpload => "The specified multipart upload does not exist",
|
Self::NoSuchUpload => "The specified multipart upload does not exist",
|
||||||
Self::NoSuchVersion => "The specified version does not exist",
|
Self::NoSuchVersion => "The specified version does not exist",
|
||||||
Self::NoSuchTagSet => "The TagSet does not exist",
|
Self::NoSuchTagSet => "The TagSet does not exist",
|
||||||
|
Self::ObjectCorrupted => "The stored object is corrupted and cannot be served",
|
||||||
Self::PreconditionFailed => "At least one of the preconditions you specified did not hold",
|
Self::PreconditionFailed => "At least one of the preconditions you specified did not hold",
|
||||||
Self::NotModified => "Not Modified",
|
Self::NotModified => "Not Modified",
|
||||||
Self::QuotaExceeded => "The bucket quota has been exceeded",
|
Self::QuotaExceeded => "The bucket quota has been exceeded",
|
||||||
|
Self::RequestTimeTooSkewed => "The difference between the request time and the server's time is too large",
|
||||||
|
Self::ServerSideEncryptionConfigurationNotFoundError => "The server side encryption configuration was not found",
|
||||||
Self::SignatureDoesNotMatch => "The request signature we calculated does not match the signature you provided",
|
Self::SignatureDoesNotMatch => "The request signature we calculated does not match the signature you provided",
|
||||||
Self::SlowDown => "Please reduce your request rate",
|
Self::SlowDown => "Please reduce your request rate",
|
||||||
}
|
}
|
||||||
@@ -168,6 +206,7 @@ impl S3Error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn to_xml(&self) -> String {
|
pub fn to_xml(&self) -> String {
|
||||||
|
let host_id = derive_host_id(&self.request_id);
|
||||||
format!(
|
format!(
|
||||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||||
<Error>\
|
<Error>\
|
||||||
@@ -175,15 +214,30 @@ impl S3Error {
|
|||||||
<Message>{}</Message>\
|
<Message>{}</Message>\
|
||||||
<Resource>{}</Resource>\
|
<Resource>{}</Resource>\
|
||||||
<RequestId>{}</RequestId>\
|
<RequestId>{}</RequestId>\
|
||||||
|
<HostId>{}</HostId>\
|
||||||
</Error>",
|
</Error>",
|
||||||
self.code.as_str(),
|
self.code.as_str(),
|
||||||
xml_escape(&self.message),
|
xml_escape(&self.message),
|
||||||
xml_escape(&self.resource),
|
xml_escape(&self.resource),
|
||||||
xml_escape(&self.request_id),
|
xml_escape(&self.request_id),
|
||||||
|
xml_escape(&host_id),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn derive_host_id(request_id: &str) -> String {
|
||||||
|
use base64::engine::general_purpose::STANDARD as B64;
|
||||||
|
use base64::Engine;
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
if request_id.is_empty() {
|
||||||
|
return String::new();
|
||||||
|
}
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
hasher.update(b"myfsio-host-id\0");
|
||||||
|
hasher.update(request_id.as_bytes());
|
||||||
|
B64.encode(hasher.finalize())
|
||||||
|
}
|
||||||
|
|
||||||
impl fmt::Display for S3Error {
|
impl fmt::Display for S3Error {
|
||||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
write!(f, "{}: {}", self.code, self.message)
|
write!(f, "{}: {}", self.code, self.message)
|
||||||
|
|||||||
@@ -12,6 +12,12 @@ pub struct ObjectMeta {
|
|||||||
pub content_type: Option<String>,
|
pub content_type: Option<String>,
|
||||||
pub storage_class: Option<String>,
|
pub storage_class: Option<String>,
|
||||||
pub metadata: HashMap<String, String>,
|
pub metadata: HashMap<String, String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub version_id: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub is_delete_marker: bool,
|
||||||
|
#[serde(default, skip_serializing)]
|
||||||
|
pub internal_metadata: HashMap<String, String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ObjectMeta {
|
impl ObjectMeta {
|
||||||
@@ -24,10 +30,20 @@ impl ObjectMeta {
|
|||||||
content_type: None,
|
content_type: None,
|
||||||
storage_class: Some("STANDARD".to_string()),
|
storage_class: Some("STANDARD".to_string()),
|
||||||
metadata: HashMap::new(),
|
metadata: HashMap::new(),
|
||||||
|
version_id: None,
|
||||||
|
is_delete_marker: false,
|
||||||
|
internal_metadata: HashMap::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
|
pub struct DeleteOutcome {
|
||||||
|
pub version_id: Option<String>,
|
||||||
|
pub is_delete_marker: bool,
|
||||||
|
pub existed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct BucketMeta {
|
pub struct BucketMeta {
|
||||||
pub name: String,
|
pub name: String,
|
||||||
@@ -122,11 +138,34 @@ pub struct Tag {
|
|||||||
pub value: String,
|
pub value: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||||
|
pub enum VersioningStatus {
|
||||||
|
#[default]
|
||||||
|
Disabled,
|
||||||
|
Enabled,
|
||||||
|
Suspended,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VersioningStatus {
|
||||||
|
pub fn is_enabled(self) -> bool {
|
||||||
|
matches!(self, VersioningStatus::Enabled)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_active(self) -> bool {
|
||||||
|
matches!(
|
||||||
|
self,
|
||||||
|
VersioningStatus::Enabled | VersioningStatus::Suspended
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||||
pub struct BucketConfig {
|
pub struct BucketConfig {
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub versioning_enabled: bool,
|
pub versioning_enabled: bool,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
|
pub versioning_suspended: bool,
|
||||||
|
#[serde(default)]
|
||||||
pub tags: Vec<Tag>,
|
pub tags: Vec<Tag>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub cors: Option<serde_json::Value>,
|
pub cors: Option<serde_json::Value>,
|
||||||
@@ -152,6 +191,35 @@ pub struct BucketConfig {
|
|||||||
pub replication: Option<serde_json::Value>,
|
pub replication: Option<serde_json::Value>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl BucketConfig {
|
||||||
|
pub fn versioning_status(&self) -> VersioningStatus {
|
||||||
|
if self.versioning_enabled {
|
||||||
|
VersioningStatus::Enabled
|
||||||
|
} else if self.versioning_suspended {
|
||||||
|
VersioningStatus::Suspended
|
||||||
|
} else {
|
||||||
|
VersioningStatus::Disabled
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_versioning_status(&mut self, status: VersioningStatus) {
|
||||||
|
match status {
|
||||||
|
VersioningStatus::Enabled => {
|
||||||
|
self.versioning_enabled = true;
|
||||||
|
self.versioning_suspended = false;
|
||||||
|
}
|
||||||
|
VersioningStatus::Suspended => {
|
||||||
|
self.versioning_enabled = false;
|
||||||
|
self.versioning_suspended = true;
|
||||||
|
}
|
||||||
|
VersioningStatus::Disabled => {
|
||||||
|
self.versioning_enabled = false;
|
||||||
|
self.versioning_suspended = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct QuotaConfig {
|
pub struct QuotaConfig {
|
||||||
pub max_bytes: Option<u64>,
|
pub max_bytes: Option<u64>,
|
||||||
|
|||||||
@@ -145,6 +145,113 @@ pub fn decrypt_stream_chunked(
|
|||||||
Ok(chunk_count)
|
Ok(chunk_count)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const GCM_TAG_LEN: usize = 16;
|
||||||
|
|
||||||
|
pub fn decrypt_stream_chunked_range(
|
||||||
|
input_path: &Path,
|
||||||
|
output_path: &Path,
|
||||||
|
key: &[u8],
|
||||||
|
base_nonce: &[u8],
|
||||||
|
chunk_plain_size: usize,
|
||||||
|
plaintext_size: u64,
|
||||||
|
plain_start: u64,
|
||||||
|
plain_end_inclusive: u64,
|
||||||
|
) -> Result<u64, CryptoError> {
|
||||||
|
if key.len() != 32 {
|
||||||
|
return Err(CryptoError::InvalidKeySize(key.len()));
|
||||||
|
}
|
||||||
|
if base_nonce.len() != 12 {
|
||||||
|
return Err(CryptoError::InvalidNonceSize(base_nonce.len()));
|
||||||
|
}
|
||||||
|
if chunk_plain_size == 0 {
|
||||||
|
return Err(CryptoError::EncryptionFailed(
|
||||||
|
"chunk_plain_size must be > 0".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if plaintext_size == 0 {
|
||||||
|
let _ = File::create(output_path)?;
|
||||||
|
return Ok(0);
|
||||||
|
}
|
||||||
|
if plain_start > plain_end_inclusive || plain_end_inclusive >= plaintext_size {
|
||||||
|
return Err(CryptoError::EncryptionFailed(format!(
|
||||||
|
"range [{}, {}] invalid for plaintext size {}",
|
||||||
|
plain_start, plain_end_inclusive, plaintext_size
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let key_arr: [u8; 32] = key.try_into().unwrap();
|
||||||
|
let nonce_arr: [u8; 12] = base_nonce.try_into().unwrap();
|
||||||
|
let cipher = Aes256Gcm::new(&key_arr.into());
|
||||||
|
|
||||||
|
let n = chunk_plain_size as u64;
|
||||||
|
let first_chunk = (plain_start / n) as u32;
|
||||||
|
let last_chunk = (plain_end_inclusive / n) as u32;
|
||||||
|
let total_chunks = plaintext_size.div_ceil(n) as u32;
|
||||||
|
let final_chunk_plain = plaintext_size - (total_chunks as u64 - 1) * n;
|
||||||
|
|
||||||
|
let mut infile = File::open(input_path)?;
|
||||||
|
|
||||||
|
let mut header = [0u8; HEADER_SIZE];
|
||||||
|
infile.read_exact(&mut header)?;
|
||||||
|
let stored_chunk_count = u32::from_be_bytes(header);
|
||||||
|
if stored_chunk_count != total_chunks {
|
||||||
|
return Err(CryptoError::EncryptionFailed(format!(
|
||||||
|
"chunk count mismatch: header says {}, plaintext_size implies {}",
|
||||||
|
stored_chunk_count, total_chunks
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut outfile = File::create(output_path)?;
|
||||||
|
|
||||||
|
let stride = n + GCM_TAG_LEN as u64 + HEADER_SIZE as u64;
|
||||||
|
let first_offset = HEADER_SIZE as u64 + first_chunk as u64 * stride;
|
||||||
|
infile.seek(SeekFrom::Start(first_offset))?;
|
||||||
|
|
||||||
|
let mut size_buf = [0u8; HEADER_SIZE];
|
||||||
|
let mut bytes_written: u64 = 0;
|
||||||
|
|
||||||
|
for chunk_index in first_chunk..=last_chunk {
|
||||||
|
infile.read_exact(&mut size_buf)?;
|
||||||
|
let ct_len = u32::from_be_bytes(size_buf) as usize;
|
||||||
|
|
||||||
|
let expected_plain = if chunk_index + 1 == total_chunks {
|
||||||
|
final_chunk_plain as usize
|
||||||
|
} else {
|
||||||
|
chunk_plain_size
|
||||||
|
};
|
||||||
|
let expected_ct = expected_plain + GCM_TAG_LEN;
|
||||||
|
if ct_len != expected_ct {
|
||||||
|
return Err(CryptoError::EncryptionFailed(format!(
|
||||||
|
"chunk {} stored length {} != expected {} (corrupt file or chunk_size mismatch)",
|
||||||
|
chunk_index, ct_len, expected_ct
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut encrypted = vec![0u8; ct_len];
|
||||||
|
infile.read_exact(&mut encrypted)?;
|
||||||
|
|
||||||
|
let nonce_bytes = derive_chunk_nonce(&nonce_arr, chunk_index)?;
|
||||||
|
let nonce = Nonce::from_slice(&nonce_bytes);
|
||||||
|
let decrypted = cipher
|
||||||
|
.decrypt(nonce, encrypted.as_ref())
|
||||||
|
.map_err(|_| CryptoError::DecryptionFailed(chunk_index))?;
|
||||||
|
|
||||||
|
let chunk_plain_start = chunk_index as u64 * n;
|
||||||
|
let chunk_plain_end_exclusive = chunk_plain_start + decrypted.len() as u64;
|
||||||
|
|
||||||
|
let slice_start = plain_start.saturating_sub(chunk_plain_start) as usize;
|
||||||
|
let slice_end = (plain_end_inclusive + 1).min(chunk_plain_end_exclusive);
|
||||||
|
let slice_end_local = (slice_end - chunk_plain_start) as usize;
|
||||||
|
|
||||||
|
if slice_end_local > slice_start {
|
||||||
|
outfile.write_all(&decrypted[slice_start..slice_end_local])?;
|
||||||
|
bytes_written += (slice_end_local - slice_start) as u64;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(bytes_written)
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn encrypt_stream_chunked_async(
|
pub async fn encrypt_stream_chunked_async(
|
||||||
input_path: &Path,
|
input_path: &Path,
|
||||||
output_path: &Path,
|
output_path: &Path,
|
||||||
@@ -230,6 +337,191 @@ mod tests {
|
|||||||
assert!(matches!(result, Err(CryptoError::InvalidKeySize(16))));
|
assert!(matches!(result, Err(CryptoError::InvalidKeySize(16))));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn write_file(path: &Path, data: &[u8]) {
|
||||||
|
std::fs::File::create(path).unwrap().write_all(data).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn make_encrypted_file(
|
||||||
|
dir: &Path,
|
||||||
|
data: &[u8],
|
||||||
|
key: &[u8; 32],
|
||||||
|
nonce: &[u8; 12],
|
||||||
|
chunk: usize,
|
||||||
|
) -> std::path::PathBuf {
|
||||||
|
let input = dir.join("input.bin");
|
||||||
|
let encrypted = dir.join("encrypted.bin");
|
||||||
|
write_file(&input, data);
|
||||||
|
encrypt_stream_chunked(&input, &encrypted, key, nonce, Some(chunk)).unwrap();
|
||||||
|
encrypted
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_range_within_single_chunk() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let data: Vec<u8> = (0u8..=255).cycle().take(4096).collect();
|
||||||
|
let key = [0x33u8; 32];
|
||||||
|
let nonce = [0x07u8; 12];
|
||||||
|
let encrypted = make_encrypted_file(dir.path(), &data, &key, &nonce, 1024);
|
||||||
|
let out = dir.path().join("range.bin");
|
||||||
|
|
||||||
|
let n = decrypt_stream_chunked_range(
|
||||||
|
&encrypted,
|
||||||
|
&out,
|
||||||
|
&key,
|
||||||
|
&nonce,
|
||||||
|
1024,
|
||||||
|
data.len() as u64,
|
||||||
|
200,
|
||||||
|
399,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(n, 200);
|
||||||
|
let got = std::fs::read(&out).unwrap();
|
||||||
|
assert_eq!(got, &data[200..400]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_range_spanning_multiple_chunks() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let data: Vec<u8> = (0..5000u32).map(|i| (i % 251) as u8).collect();
|
||||||
|
let key = [0x44u8; 32];
|
||||||
|
let nonce = [0x02u8; 12];
|
||||||
|
let encrypted = make_encrypted_file(dir.path(), &data, &key, &nonce, 512);
|
||||||
|
let out = dir.path().join("range.bin");
|
||||||
|
|
||||||
|
let n = decrypt_stream_chunked_range(
|
||||||
|
&encrypted,
|
||||||
|
&out,
|
||||||
|
&key,
|
||||||
|
&nonce,
|
||||||
|
512,
|
||||||
|
data.len() as u64,
|
||||||
|
100,
|
||||||
|
2999,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(n, 2900);
|
||||||
|
let got = std::fs::read(&out).unwrap();
|
||||||
|
assert_eq!(got, &data[100..3000]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_range_covers_final_partial_chunk() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let data: Vec<u8> = (0..1300u32).map(|i| (i % 71) as u8).collect();
|
||||||
|
let key = [0x55u8; 32];
|
||||||
|
let nonce = [0x0au8; 12];
|
||||||
|
let encrypted = make_encrypted_file(dir.path(), &data, &key, &nonce, 512);
|
||||||
|
let out = dir.path().join("range.bin");
|
||||||
|
|
||||||
|
let n = decrypt_stream_chunked_range(
|
||||||
|
&encrypted,
|
||||||
|
&out,
|
||||||
|
&key,
|
||||||
|
&nonce,
|
||||||
|
512,
|
||||||
|
data.len() as u64,
|
||||||
|
900,
|
||||||
|
1299,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(n, 400);
|
||||||
|
let got = std::fs::read(&out).unwrap();
|
||||||
|
assert_eq!(got, &data[900..1300]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_range_full_object() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let data: Vec<u8> = (0..2048u32).map(|i| (i % 13) as u8).collect();
|
||||||
|
let key = [0x11u8; 32];
|
||||||
|
let nonce = [0x33u8; 12];
|
||||||
|
let encrypted = make_encrypted_file(dir.path(), &data, &key, &nonce, 512);
|
||||||
|
let out = dir.path().join("range.bin");
|
||||||
|
|
||||||
|
let n = decrypt_stream_chunked_range(
|
||||||
|
&encrypted,
|
||||||
|
&out,
|
||||||
|
&key,
|
||||||
|
&nonce,
|
||||||
|
512,
|
||||||
|
data.len() as u64,
|
||||||
|
0,
|
||||||
|
data.len() as u64 - 1,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(n, data.len() as u64);
|
||||||
|
let got = std::fs::read(&out).unwrap();
|
||||||
|
assert_eq!(got, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_range_wrong_key_fails() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let data = b"range-auth-check".repeat(100);
|
||||||
|
let key = [0x66u8; 32];
|
||||||
|
let nonce = [0x09u8; 12];
|
||||||
|
let encrypted = make_encrypted_file(dir.path(), &data, &key, &nonce, 256);
|
||||||
|
let out = dir.path().join("range.bin");
|
||||||
|
|
||||||
|
let wrong = [0x67u8; 32];
|
||||||
|
let r = decrypt_stream_chunked_range(
|
||||||
|
&encrypted,
|
||||||
|
&out,
|
||||||
|
&wrong,
|
||||||
|
&nonce,
|
||||||
|
256,
|
||||||
|
data.len() as u64,
|
||||||
|
0,
|
||||||
|
data.len() as u64 - 1,
|
||||||
|
);
|
||||||
|
assert!(matches!(r, Err(CryptoError::DecryptionFailed(_))));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_range_out_of_bounds_rejected() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let data = vec![0u8; 100];
|
||||||
|
let key = [0x22u8; 32];
|
||||||
|
let nonce = [0x44u8; 12];
|
||||||
|
let encrypted = make_encrypted_file(dir.path(), &data, &key, &nonce, 64);
|
||||||
|
let out = dir.path().join("range.bin");
|
||||||
|
|
||||||
|
let r = decrypt_stream_chunked_range(
|
||||||
|
&encrypted,
|
||||||
|
&out,
|
||||||
|
&key,
|
||||||
|
&nonce,
|
||||||
|
64,
|
||||||
|
data.len() as u64,
|
||||||
|
50,
|
||||||
|
200,
|
||||||
|
);
|
||||||
|
assert!(r.is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_range_mismatched_chunk_size_detected() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let data: Vec<u8> = (0..2048u32).map(|i| i as u8).collect();
|
||||||
|
let key = [0x77u8; 32];
|
||||||
|
let nonce = [0x88u8; 12];
|
||||||
|
let encrypted = make_encrypted_file(dir.path(), &data, &key, &nonce, 512);
|
||||||
|
let out = dir.path().join("range.bin");
|
||||||
|
|
||||||
|
let r = decrypt_stream_chunked_range(
|
||||||
|
&encrypted,
|
||||||
|
&out,
|
||||||
|
&key,
|
||||||
|
&nonce,
|
||||||
|
1024,
|
||||||
|
data.len() as u64,
|
||||||
|
0,
|
||||||
|
1023,
|
||||||
|
);
|
||||||
|
assert!(r.is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_wrong_key_fails_decrypt() {
|
fn test_wrong_key_fails_decrypt() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
|||||||
@@ -4,7 +4,9 @@ use rand::RngCore;
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use crate::aes_gcm::{decrypt_stream_chunked, encrypt_stream_chunked, CryptoError};
|
use crate::aes_gcm::{
|
||||||
|
decrypt_stream_chunked, decrypt_stream_chunked_range, encrypt_stream_chunked, CryptoError,
|
||||||
|
};
|
||||||
use crate::kms::KmsService;
|
use crate::kms::KmsService;
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq)]
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
@@ -37,6 +39,8 @@ pub struct EncryptionMetadata {
|
|||||||
pub nonce: String,
|
pub nonce: String,
|
||||||
pub encrypted_data_key: Option<String>,
|
pub encrypted_data_key: Option<String>,
|
||||||
pub kms_key_id: Option<String>,
|
pub kms_key_id: Option<String>,
|
||||||
|
pub chunk_size: Option<usize>,
|
||||||
|
pub plaintext_size: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl EncryptionMetadata {
|
impl EncryptionMetadata {
|
||||||
@@ -53,6 +57,15 @@ impl EncryptionMetadata {
|
|||||||
if let Some(ref kid) = self.kms_key_id {
|
if let Some(ref kid) = self.kms_key_id {
|
||||||
map.insert("x-amz-encryption-key-id".to_string(), kid.clone());
|
map.insert("x-amz-encryption-key-id".to_string(), kid.clone());
|
||||||
}
|
}
|
||||||
|
if let Some(cs) = self.chunk_size {
|
||||||
|
map.insert("x-amz-encryption-chunk-size".to_string(), cs.to_string());
|
||||||
|
}
|
||||||
|
if let Some(ps) = self.plaintext_size {
|
||||||
|
map.insert(
|
||||||
|
"x-amz-encryption-plaintext-size".to_string(),
|
||||||
|
ps.to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
map
|
map
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -64,6 +77,12 @@ impl EncryptionMetadata {
|
|||||||
nonce: nonce.clone(),
|
nonce: nonce.clone(),
|
||||||
encrypted_data_key: meta.get("x-amz-encrypted-data-key").cloned(),
|
encrypted_data_key: meta.get("x-amz-encrypted-data-key").cloned(),
|
||||||
kms_key_id: meta.get("x-amz-encryption-key-id").cloned(),
|
kms_key_id: meta.get("x-amz-encryption-key-id").cloned(),
|
||||||
|
chunk_size: meta
|
||||||
|
.get("x-amz-encryption-chunk-size")
|
||||||
|
.and_then(|s| s.parse().ok()),
|
||||||
|
plaintext_size: meta
|
||||||
|
.get("x-amz-encryption-plaintext-size")
|
||||||
|
.and_then(|s| s.parse().ok()),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -76,6 +95,8 @@ impl EncryptionMetadata {
|
|||||||
meta.remove("x-amz-encryption-nonce");
|
meta.remove("x-amz-encryption-nonce");
|
||||||
meta.remove("x-amz-encrypted-data-key");
|
meta.remove("x-amz-encrypted-data-key");
|
||||||
meta.remove("x-amz-encryption-key-id");
|
meta.remove("x-amz-encryption-key-id");
|
||||||
|
meta.remove("x-amz-encryption-chunk-size");
|
||||||
|
meta.remove("x-amz-encryption-plaintext-size");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -212,6 +233,11 @@ impl EncryptionService {
|
|||||||
data_key
|
data_key
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let plaintext_size = tokio::fs::metadata(input_path)
|
||||||
|
.await
|
||||||
|
.map_err(CryptoError::Io)?
|
||||||
|
.len();
|
||||||
|
|
||||||
let ip = input_path.to_owned();
|
let ip = input_path.to_owned();
|
||||||
let op = output_path.to_owned();
|
let op = output_path.to_owned();
|
||||||
let ak = actual_key;
|
let ak = actual_key;
|
||||||
@@ -228,22 +254,23 @@ impl EncryptionService {
|
|||||||
nonce: B64.encode(nonce),
|
nonce: B64.encode(nonce),
|
||||||
encrypted_data_key,
|
encrypted_data_key,
|
||||||
kms_key_id,
|
kms_key_id,
|
||||||
|
chunk_size: Some(chunk_size),
|
||||||
|
plaintext_size: Some(plaintext_size),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn decrypt_object(
|
async fn resolve_data_key(
|
||||||
&self,
|
&self,
|
||||||
input_path: &Path,
|
|
||||||
output_path: &Path,
|
|
||||||
enc_meta: &EncryptionMetadata,
|
enc_meta: &EncryptionMetadata,
|
||||||
customer_key: Option<&[u8]>,
|
customer_key: Option<&[u8]>,
|
||||||
) -> Result<(), CryptoError> {
|
) -> Result<([u8; 32], [u8; 12]), CryptoError> {
|
||||||
let nonce_bytes = B64
|
let nonce_bytes = B64
|
||||||
.decode(&enc_meta.nonce)
|
.decode(&enc_meta.nonce)
|
||||||
.map_err(|e| CryptoError::EncryptionFailed(format!("Bad nonce encoding: {}", e)))?;
|
.map_err(|e| CryptoError::EncryptionFailed(format!("Bad nonce encoding: {}", e)))?;
|
||||||
if nonce_bytes.len() != 12 {
|
if nonce_bytes.len() != 12 {
|
||||||
return Err(CryptoError::InvalidNonceSize(nonce_bytes.len()));
|
return Err(CryptoError::InvalidNonceSize(nonce_bytes.len()));
|
||||||
}
|
}
|
||||||
|
let nonce: [u8; 12] = nonce_bytes.try_into().unwrap();
|
||||||
|
|
||||||
let data_key: [u8; 32] = if let Some(ck) = customer_key {
|
let data_key: [u8; 32] = if let Some(ck) = customer_key {
|
||||||
if ck.len() != 32 {
|
if ck.len() != 32 {
|
||||||
@@ -281,15 +308,62 @@ impl EncryptionService {
|
|||||||
self.unwrap_data_key(wrapped)?
|
self.unwrap_data_key(wrapped)?
|
||||||
};
|
};
|
||||||
|
|
||||||
|
Ok((data_key, nonce))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn decrypt_object(
|
||||||
|
&self,
|
||||||
|
input_path: &Path,
|
||||||
|
output_path: &Path,
|
||||||
|
enc_meta: &EncryptionMetadata,
|
||||||
|
customer_key: Option<&[u8]>,
|
||||||
|
) -> Result<(), CryptoError> {
|
||||||
|
let (data_key, nonce) = self.resolve_data_key(enc_meta, customer_key).await?;
|
||||||
|
|
||||||
let ip = input_path.to_owned();
|
let ip = input_path.to_owned();
|
||||||
let op = output_path.to_owned();
|
let op = output_path.to_owned();
|
||||||
let nb: [u8; 12] = nonce_bytes.try_into().unwrap();
|
tokio::task::spawn_blocking(move || decrypt_stream_chunked(&ip, &op, &data_key, &nonce))
|
||||||
tokio::task::spawn_blocking(move || decrypt_stream_chunked(&ip, &op, &data_key, &nb))
|
|
||||||
.await
|
.await
|
||||||
.map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))??;
|
.map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))??;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn decrypt_object_range(
|
||||||
|
&self,
|
||||||
|
input_path: &Path,
|
||||||
|
output_path: &Path,
|
||||||
|
enc_meta: &EncryptionMetadata,
|
||||||
|
customer_key: Option<&[u8]>,
|
||||||
|
plain_start: u64,
|
||||||
|
plain_end_inclusive: u64,
|
||||||
|
) -> Result<u64, CryptoError> {
|
||||||
|
let chunk_size = enc_meta.chunk_size.ok_or_else(|| {
|
||||||
|
CryptoError::EncryptionFailed("chunk_size missing from encryption metadata".into())
|
||||||
|
})?;
|
||||||
|
let plaintext_size = enc_meta.plaintext_size.ok_or_else(|| {
|
||||||
|
CryptoError::EncryptionFailed("plaintext_size missing from encryption metadata".into())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let (data_key, nonce) = self.resolve_data_key(enc_meta, customer_key).await?;
|
||||||
|
|
||||||
|
let ip = input_path.to_owned();
|
||||||
|
let op = output_path.to_owned();
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
decrypt_stream_chunked_range(
|
||||||
|
&ip,
|
||||||
|
&op,
|
||||||
|
&data_key,
|
||||||
|
&nonce,
|
||||||
|
chunk_size,
|
||||||
|
plaintext_size,
|
||||||
|
plain_start,
|
||||||
|
plain_end_inclusive,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))?
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -383,12 +457,26 @@ mod tests {
|
|||||||
nonce: "dGVzdG5vbmNlMTI=".to_string(),
|
nonce: "dGVzdG5vbmNlMTI=".to_string(),
|
||||||
encrypted_data_key: Some("c29tZWtleQ==".to_string()),
|
encrypted_data_key: Some("c29tZWtleQ==".to_string()),
|
||||||
kms_key_id: None,
|
kms_key_id: None,
|
||||||
|
chunk_size: Some(65_536),
|
||||||
|
plaintext_size: Some(1_234_567),
|
||||||
};
|
};
|
||||||
let map = meta.to_metadata_map();
|
let map = meta.to_metadata_map();
|
||||||
let restored = EncryptionMetadata::from_metadata(&map).unwrap();
|
let restored = EncryptionMetadata::from_metadata(&map).unwrap();
|
||||||
assert_eq!(restored.algorithm, "AES256");
|
assert_eq!(restored.algorithm, "AES256");
|
||||||
assert_eq!(restored.nonce, meta.nonce);
|
assert_eq!(restored.nonce, meta.nonce);
|
||||||
assert_eq!(restored.encrypted_data_key, meta.encrypted_data_key);
|
assert_eq!(restored.encrypted_data_key, meta.encrypted_data_key);
|
||||||
|
assert_eq!(restored.chunk_size, Some(65_536));
|
||||||
|
assert_eq!(restored.plaintext_size, Some(1_234_567));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_encryption_metadata_legacy_missing_sizes() {
|
||||||
|
let mut map = HashMap::new();
|
||||||
|
map.insert("x-amz-server-side-encryption".to_string(), "AES256".into());
|
||||||
|
map.insert("x-amz-encryption-nonce".to_string(), "aGVsbG8=".into());
|
||||||
|
let restored = EncryptionMetadata::from_metadata(&map).unwrap();
|
||||||
|
assert_eq!(restored.chunk_size, None);
|
||||||
|
assert_eq!(restored.plaintext_size, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -23,15 +23,19 @@ serde_urlencoded = "0.7"
|
|||||||
tracing = { workspace = true }
|
tracing = { workspace = true }
|
||||||
tracing-subscriber = { workspace = true }
|
tracing-subscriber = { workspace = true }
|
||||||
tokio-util = { workspace = true }
|
tokio-util = { workspace = true }
|
||||||
|
tokio-stream = { workspace = true }
|
||||||
chrono = { workspace = true }
|
chrono = { workspace = true }
|
||||||
|
chrono-tz = { workspace = true }
|
||||||
uuid = { workspace = true }
|
uuid = { workspace = true }
|
||||||
futures = { workspace = true }
|
futures = { workspace = true }
|
||||||
|
http-body = "1"
|
||||||
http-body-util = "0.1"
|
http-body-util = "0.1"
|
||||||
percent-encoding = { workspace = true }
|
percent-encoding = { workspace = true }
|
||||||
quick-xml = { workspace = true }
|
quick-xml = { workspace = true }
|
||||||
mime_guess = "2"
|
mime_guess = "2"
|
||||||
crc32fast = { workspace = true }
|
crc32fast = { workspace = true }
|
||||||
sha2 = { workspace = true }
|
sha2 = { workspace = true }
|
||||||
|
hex = { workspace = true }
|
||||||
duckdb = { workspace = true }
|
duckdb = { workspace = true }
|
||||||
roxmltree = "0.20"
|
roxmltree = "0.20"
|
||||||
parking_lot = { workspace = true }
|
parking_lot = { workspace = true }
|
||||||
@@ -45,6 +49,7 @@ aws-smithy-types = { workspace = true }
|
|||||||
async-trait = { workspace = true }
|
async-trait = { workspace = true }
|
||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
tera = { workspace = true }
|
tera = { workspace = true }
|
||||||
|
rust-embed = { version = "8", features = ["debug-embed", "include-exclude", "interpolate-folder-path"] }
|
||||||
cookie = { workspace = true }
|
cookie = { workspace = true }
|
||||||
subtle = { workspace = true }
|
subtle = { workspace = true }
|
||||||
clap = { workspace = true }
|
clap = { workspace = true }
|
||||||
|
|||||||
@@ -39,6 +39,12 @@ pub struct ServerConfig {
|
|||||||
pub gc_lock_file_max_age_hours: f64,
|
pub gc_lock_file_max_age_hours: f64,
|
||||||
pub gc_dry_run: bool,
|
pub gc_dry_run: bool,
|
||||||
pub integrity_enabled: bool,
|
pub integrity_enabled: bool,
|
||||||
|
pub integrity_interval_hours: f64,
|
||||||
|
pub integrity_batch_size: usize,
|
||||||
|
pub integrity_auto_heal: bool,
|
||||||
|
pub integrity_dry_run: bool,
|
||||||
|
pub integrity_heal_concurrency: usize,
|
||||||
|
pub integrity_quarantine_retention_days: u64,
|
||||||
pub metrics_enabled: bool,
|
pub metrics_enabled: bool,
|
||||||
pub metrics_history_enabled: bool,
|
pub metrics_history_enabled: bool,
|
||||||
pub metrics_interval_minutes: u64,
|
pub metrics_interval_minutes: u64,
|
||||||
@@ -78,10 +84,16 @@ pub struct ServerConfig {
|
|||||||
pub cors_expose_headers: Vec<String>,
|
pub cors_expose_headers: Vec<String>,
|
||||||
pub session_lifetime_days: u64,
|
pub session_lifetime_days: u64,
|
||||||
pub log_level: String,
|
pub log_level: String,
|
||||||
|
pub display_timezone: String,
|
||||||
pub multipart_min_part_size: u64,
|
pub multipart_min_part_size: u64,
|
||||||
pub bulk_delete_max_keys: usize,
|
pub bulk_delete_max_keys: usize,
|
||||||
pub stream_chunk_size: usize,
|
pub stream_chunk_size: usize,
|
||||||
|
pub request_body_timeout_secs: u64,
|
||||||
pub ratelimit_default: RateLimitSetting,
|
pub ratelimit_default: RateLimitSetting,
|
||||||
|
pub ratelimit_list_buckets: RateLimitSetting,
|
||||||
|
pub ratelimit_bucket_ops: RateLimitSetting,
|
||||||
|
pub ratelimit_object_ops: RateLimitSetting,
|
||||||
|
pub ratelimit_head_ops: RateLimitSetting,
|
||||||
pub ratelimit_admin: RateLimitSetting,
|
pub ratelimit_admin: RateLimitSetting,
|
||||||
pub ratelimit_storage_uri: String,
|
pub ratelimit_storage_uri: String,
|
||||||
pub ui_enabled: bool,
|
pub ui_enabled: bool,
|
||||||
@@ -163,6 +175,13 @@ impl ServerConfig {
|
|||||||
let gc_dry_run = parse_bool_env("GC_DRY_RUN", false);
|
let gc_dry_run = parse_bool_env("GC_DRY_RUN", false);
|
||||||
|
|
||||||
let integrity_enabled = parse_bool_env("INTEGRITY_ENABLED", false);
|
let integrity_enabled = parse_bool_env("INTEGRITY_ENABLED", false);
|
||||||
|
let integrity_interval_hours = parse_f64_env("INTEGRITY_INTERVAL_HOURS", 24.0);
|
||||||
|
let integrity_batch_size = parse_usize_env("INTEGRITY_BATCH_SIZE", 10_000);
|
||||||
|
let integrity_auto_heal = parse_bool_env("INTEGRITY_AUTO_HEAL", false);
|
||||||
|
let integrity_dry_run = parse_bool_env("INTEGRITY_DRY_RUN", false);
|
||||||
|
let integrity_heal_concurrency = parse_usize_env("INTEGRITY_HEAL_CONCURRENCY", 4);
|
||||||
|
let integrity_quarantine_retention_days =
|
||||||
|
parse_u64_env("INTEGRITY_QUARANTINE_RETENTION_DAYS", 7);
|
||||||
|
|
||||||
let metrics_enabled = parse_bool_env("OPERATION_METRICS_ENABLED", false);
|
let metrics_enabled = parse_bool_env("OPERATION_METRICS_ENABLED", false);
|
||||||
|
|
||||||
@@ -222,11 +241,30 @@ impl ServerConfig {
|
|||||||
let cors_expose_headers = parse_list_env("CORS_EXPOSE_HEADERS", "*");
|
let cors_expose_headers = parse_list_env("CORS_EXPOSE_HEADERS", "*");
|
||||||
let session_lifetime_days = parse_u64_env("SESSION_LIFETIME_DAYS", 1);
|
let session_lifetime_days = parse_u64_env("SESSION_LIFETIME_DAYS", 1);
|
||||||
let log_level = std::env::var("LOG_LEVEL").unwrap_or_else(|_| "INFO".to_string());
|
let log_level = std::env::var("LOG_LEVEL").unwrap_or_else(|_| "INFO".to_string());
|
||||||
|
let display_timezone = {
|
||||||
|
let raw = std::env::var("DISPLAY_TIMEZONE").unwrap_or_else(|_| "UTC".to_string());
|
||||||
|
match raw.parse::<chrono_tz::Tz>() {
|
||||||
|
Ok(_) => raw,
|
||||||
|
Err(_) => {
|
||||||
|
tracing::warn!(
|
||||||
|
"Invalid DISPLAY_TIMEZONE '{}', falling back to UTC",
|
||||||
|
raw
|
||||||
|
);
|
||||||
|
"UTC".to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
let multipart_min_part_size = parse_u64_env("MULTIPART_MIN_PART_SIZE", 5_242_880);
|
let multipart_min_part_size = parse_u64_env("MULTIPART_MIN_PART_SIZE", 5_242_880);
|
||||||
let bulk_delete_max_keys = parse_usize_env("BULK_DELETE_MAX_KEYS", 1000);
|
let bulk_delete_max_keys = parse_usize_env("BULK_DELETE_MAX_KEYS", 1000);
|
||||||
let stream_chunk_size = parse_usize_env("STREAM_CHUNK_SIZE", 1_048_576);
|
let stream_chunk_size = parse_usize_env("STREAM_CHUNK_SIZE", 1_048_576);
|
||||||
|
let request_body_timeout_secs = parse_u64_env("REQUEST_BODY_TIMEOUT_SECONDS", 60);
|
||||||
let ratelimit_default =
|
let ratelimit_default =
|
||||||
parse_rate_limit_env("RATE_LIMIT_DEFAULT", RateLimitSetting::new(200, 60));
|
parse_rate_limit_env("RATE_LIMIT_DEFAULT", RateLimitSetting::new(5000, 60));
|
||||||
|
let ratelimit_list_buckets =
|
||||||
|
parse_rate_limit_env("RATE_LIMIT_LIST_BUCKETS", ratelimit_default);
|
||||||
|
let ratelimit_bucket_ops = parse_rate_limit_env("RATE_LIMIT_BUCKET_OPS", ratelimit_default);
|
||||||
|
let ratelimit_object_ops = parse_rate_limit_env("RATE_LIMIT_OBJECT_OPS", ratelimit_default);
|
||||||
|
let ratelimit_head_ops = parse_rate_limit_env("RATE_LIMIT_HEAD_OPS", ratelimit_default);
|
||||||
let ratelimit_admin =
|
let ratelimit_admin =
|
||||||
parse_rate_limit_env("RATE_LIMIT_ADMIN", RateLimitSetting::new(60, 60));
|
parse_rate_limit_env("RATE_LIMIT_ADMIN", RateLimitSetting::new(60, 60));
|
||||||
let ratelimit_storage_uri =
|
let ratelimit_storage_uri =
|
||||||
@@ -262,6 +300,12 @@ impl ServerConfig {
|
|||||||
gc_lock_file_max_age_hours,
|
gc_lock_file_max_age_hours,
|
||||||
gc_dry_run,
|
gc_dry_run,
|
||||||
integrity_enabled,
|
integrity_enabled,
|
||||||
|
integrity_interval_hours,
|
||||||
|
integrity_batch_size,
|
||||||
|
integrity_auto_heal,
|
||||||
|
integrity_dry_run,
|
||||||
|
integrity_heal_concurrency,
|
||||||
|
integrity_quarantine_retention_days,
|
||||||
metrics_enabled,
|
metrics_enabled,
|
||||||
metrics_history_enabled,
|
metrics_history_enabled,
|
||||||
metrics_interval_minutes,
|
metrics_interval_minutes,
|
||||||
@@ -301,10 +345,16 @@ impl ServerConfig {
|
|||||||
cors_expose_headers,
|
cors_expose_headers,
|
||||||
session_lifetime_days,
|
session_lifetime_days,
|
||||||
log_level,
|
log_level,
|
||||||
|
display_timezone,
|
||||||
multipart_min_part_size,
|
multipart_min_part_size,
|
||||||
bulk_delete_max_keys,
|
bulk_delete_max_keys,
|
||||||
stream_chunk_size,
|
stream_chunk_size,
|
||||||
|
request_body_timeout_secs,
|
||||||
ratelimit_default,
|
ratelimit_default,
|
||||||
|
ratelimit_list_buckets,
|
||||||
|
ratelimit_bucket_ops,
|
||||||
|
ratelimit_object_ops,
|
||||||
|
ratelimit_head_ops,
|
||||||
ratelimit_admin,
|
ratelimit_admin,
|
||||||
ratelimit_storage_uri,
|
ratelimit_storage_uri,
|
||||||
ui_enabled,
|
ui_enabled,
|
||||||
@@ -338,6 +388,12 @@ impl Default for ServerConfig {
|
|||||||
gc_lock_file_max_age_hours: 1.0,
|
gc_lock_file_max_age_hours: 1.0,
|
||||||
gc_dry_run: false,
|
gc_dry_run: false,
|
||||||
integrity_enabled: false,
|
integrity_enabled: false,
|
||||||
|
integrity_interval_hours: 24.0,
|
||||||
|
integrity_batch_size: 10_000,
|
||||||
|
integrity_auto_heal: false,
|
||||||
|
integrity_dry_run: false,
|
||||||
|
integrity_heal_concurrency: 4,
|
||||||
|
integrity_quarantine_retention_days: 7,
|
||||||
metrics_enabled: false,
|
metrics_enabled: false,
|
||||||
metrics_history_enabled: false,
|
metrics_history_enabled: false,
|
||||||
metrics_interval_minutes: 5,
|
metrics_interval_minutes: 5,
|
||||||
@@ -384,10 +440,16 @@ impl Default for ServerConfig {
|
|||||||
cors_expose_headers: vec!["*".to_string()],
|
cors_expose_headers: vec!["*".to_string()],
|
||||||
session_lifetime_days: 1,
|
session_lifetime_days: 1,
|
||||||
log_level: "INFO".to_string(),
|
log_level: "INFO".to_string(),
|
||||||
|
display_timezone: "UTC".to_string(),
|
||||||
multipart_min_part_size: 5_242_880,
|
multipart_min_part_size: 5_242_880,
|
||||||
bulk_delete_max_keys: 1000,
|
bulk_delete_max_keys: 1000,
|
||||||
stream_chunk_size: 1_048_576,
|
stream_chunk_size: 1_048_576,
|
||||||
ratelimit_default: RateLimitSetting::new(200, 60),
|
request_body_timeout_secs: 60,
|
||||||
|
ratelimit_default: RateLimitSetting::new(5000, 60),
|
||||||
|
ratelimit_list_buckets: RateLimitSetting::new(5000, 60),
|
||||||
|
ratelimit_bucket_ops: RateLimitSetting::new(5000, 60),
|
||||||
|
ratelimit_object_ops: RateLimitSetting::new(5000, 60),
|
||||||
|
ratelimit_head_ops: RateLimitSetting::new(5000, 60),
|
||||||
ratelimit_admin: RateLimitSetting::new(60, 60),
|
ratelimit_admin: RateLimitSetting::new(60, 60),
|
||||||
ratelimit_storage_uri: "memory://".to_string(),
|
ratelimit_storage_uri: "memory://".to_string(),
|
||||||
ui_enabled: true,
|
ui_enabled: true,
|
||||||
@@ -472,7 +534,31 @@ fn parse_list_env(key: &str, default: &str) -> Vec<String> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn parse_rate_limit(value: &str) -> Option<RateLimitSetting> {
|
pub fn parse_rate_limit(value: &str) -> Option<RateLimitSetting> {
|
||||||
let parts = value.split_whitespace().collect::<Vec<_>>();
|
let trimmed = value.trim();
|
||||||
|
if let Some((requests, window)) = trimmed.split_once('/') {
|
||||||
|
let max_requests = requests.trim().parse::<u32>().ok()?;
|
||||||
|
if max_requests == 0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let window_str = window.trim().to_ascii_lowercase();
|
||||||
|
let window_seconds = if let Ok(n) = window_str.parse::<u64>() {
|
||||||
|
if n == 0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
n
|
||||||
|
} else {
|
||||||
|
match window_str.as_str() {
|
||||||
|
"s" | "sec" | "second" | "seconds" => 1,
|
||||||
|
"m" | "min" | "minute" | "minutes" => 60,
|
||||||
|
"h" | "hr" | "hour" | "hours" => 3600,
|
||||||
|
"d" | "day" | "days" => 86_400,
|
||||||
|
_ => return None,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
return Some(RateLimitSetting::new(max_requests, window_seconds));
|
||||||
|
}
|
||||||
|
|
||||||
|
let parts = trimmed.split_whitespace().collect::<Vec<_>>();
|
||||||
if parts.len() != 3 || !parts[1].eq_ignore_ascii_case("per") {
|
if parts.len() != 3 || !parts[1].eq_ignore_ascii_case("per") {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
@@ -517,6 +603,15 @@ mod tests {
|
|||||||
parse_rate_limit("3 per hours"),
|
parse_rate_limit("3 per hours"),
|
||||||
Some(RateLimitSetting::new(3, 3600))
|
Some(RateLimitSetting::new(3, 3600))
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
parse_rate_limit("50000/60"),
|
||||||
|
Some(RateLimitSetting::new(50000, 60))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
parse_rate_limit("100/minute"),
|
||||||
|
Some(RateLimitSetting::new(100, 60))
|
||||||
|
);
|
||||||
|
assert_eq!(parse_rate_limit("0/60"), None);
|
||||||
assert_eq!(parse_rate_limit("0 per minute"), None);
|
assert_eq!(parse_rate_limit("0 per minute"), None);
|
||||||
assert_eq!(parse_rate_limit("bad"), None);
|
assert_eq!(parse_rate_limit("bad"), None);
|
||||||
}
|
}
|
||||||
@@ -532,7 +627,7 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(config.object_key_max_length_bytes, 1024);
|
assert_eq!(config.object_key_max_length_bytes, 1024);
|
||||||
assert_eq!(config.object_tag_limit, 50);
|
assert_eq!(config.object_tag_limit, 50);
|
||||||
assert_eq!(config.ratelimit_default, RateLimitSetting::new(200, 60));
|
assert_eq!(config.ratelimit_default, RateLimitSetting::new(5000, 60));
|
||||||
|
|
||||||
std::env::remove_var("OBJECT_TAG_LIMIT");
|
std::env::remove_var("OBJECT_TAG_LIMIT");
|
||||||
std::env::remove_var("RATE_LIMIT_DEFAULT");
|
std::env::remove_var("RATE_LIMIT_DEFAULT");
|
||||||
|
|||||||
25
crates/myfsio-server/src/embedded.rs
Normal file
25
crates/myfsio-server/src/embedded.rs
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
use rust_embed::{EmbeddedFile, RustEmbed};
|
||||||
|
|
||||||
|
#[derive(RustEmbed)]
|
||||||
|
#[folder = "$CARGO_MANIFEST_DIR/templates"]
|
||||||
|
#[include = "*.html"]
|
||||||
|
pub struct EmbeddedTemplates;
|
||||||
|
|
||||||
|
#[derive(RustEmbed)]
|
||||||
|
#[folder = "$CARGO_MANIFEST_DIR/static"]
|
||||||
|
pub struct EmbeddedStatic;
|
||||||
|
|
||||||
|
pub fn template_names() -> Vec<String> {
|
||||||
|
EmbeddedTemplates::iter()
|
||||||
|
.map(|c: std::borrow::Cow<'static, str>| c.into_owned())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn template_contents(name: &str) -> Option<String> {
|
||||||
|
let file = EmbeddedTemplates::get(name)?;
|
||||||
|
String::from_utf8(file.data.into_owned()).ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn static_file(path: &str) -> Option<EmbeddedFile> {
|
||||||
|
EmbeddedStatic::get(path)
|
||||||
|
}
|
||||||
@@ -345,6 +345,12 @@ pub async fn register_peer_site(
|
|||||||
.get("connection_id")
|
.get("connection_id")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.map(|s| s.to_string()),
|
.map(|s| s.to_string()),
|
||||||
|
peer_inbound_access_key: payload
|
||||||
|
.get("peer_inbound_access_key")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(|s| s.to_string()),
|
||||||
created_at: Some(chrono::Utc::now().to_rfc3339()),
|
created_at: Some(chrono::Utc::now().to_rfc3339()),
|
||||||
is_healthy: false,
|
is_healthy: false,
|
||||||
last_health_check: None,
|
last_health_check: None,
|
||||||
@@ -467,6 +473,16 @@ pub async fn update_peer_site(
|
|||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.map(|s| s.to_string())
|
.map(|s| s.to_string())
|
||||||
.or(existing.connection_id),
|
.or(existing.connection_id),
|
||||||
|
peer_inbound_access_key: if payload.get("peer_inbound_access_key").is_some() {
|
||||||
|
payload
|
||||||
|
.get("peer_inbound_access_key")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
} else {
|
||||||
|
existing.peer_inbound_access_key
|
||||||
|
},
|
||||||
created_at: existing.created_at,
|
created_at: existing.created_at,
|
||||||
is_healthy: existing.is_healthy,
|
is_healthy: existing.is_healthy,
|
||||||
last_health_check: existing.last_health_check,
|
last_health_check: existing.last_health_check,
|
||||||
@@ -1412,3 +1428,166 @@ pub async fn integrity_history(
|
|||||||
None => json_response(StatusCode::OK, serde_json::json!({"executions": []})),
|
None => json_response(StatusCode::OK, serde_json::json!({"executions": []})),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn require_admin_or_registered_peer(state: &AppState, principal: &Principal) -> Option<Response> {
|
||||||
|
if principal.is_admin {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let registry = match &state.site_registry {
|
||||||
|
Some(r) => r,
|
||||||
|
None => {
|
||||||
|
return Some(json_error(
|
||||||
|
"AccessDenied",
|
||||||
|
"Admin access required",
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
for peer in registry.list_peers() {
|
||||||
|
if peer.peer_inbound_access_key.as_deref() == Some(principal.access_key.as_str()) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(json_error(
|
||||||
|
"AccessDenied",
|
||||||
|
"Admin or registered peer required",
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn build_cluster_overview_public(state: &AppState) -> serde_json::Value {
|
||||||
|
build_cluster_overview(state).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn build_cluster_overview(state: &AppState) -> serde_json::Value {
|
||||||
|
let local_site = state
|
||||||
|
.site_registry
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|r| r.get_local_site());
|
||||||
|
|
||||||
|
let buckets = state.storage.list_buckets().await.unwrap_or_default();
|
||||||
|
let bucket_count = buckets.len() as u64;
|
||||||
|
let mut total_objects: u64 = 0;
|
||||||
|
let mut size_bytes: u64 = 0;
|
||||||
|
for b in &buckets {
|
||||||
|
if let Ok(stats) = state.storage.bucket_stats(&b.name).await {
|
||||||
|
total_objects += stats.total_objects();
|
||||||
|
size_bytes += stats.total_bytes();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let (disk_total, disk_free) =
|
||||||
|
crate::services::system_metrics::sample_disk(&state.config.storage_root);
|
||||||
|
|
||||||
|
let system = match state.system_metrics.as_ref() {
|
||||||
|
Some(svc) => {
|
||||||
|
let history = svc.get_history(Some(1)).await;
|
||||||
|
history
|
||||||
|
.last()
|
||||||
|
.map(|s| {
|
||||||
|
serde_json::json!({
|
||||||
|
"cpu_percent": s.cpu_percent,
|
||||||
|
"memory_percent": s.memory_percent,
|
||||||
|
"disk_percent": s.disk_percent,
|
||||||
|
"storage_bytes": s.storage_bytes,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|| serde_json::json!({}))
|
||||||
|
}
|
||||||
|
None => serde_json::json!({}),
|
||||||
|
};
|
||||||
|
|
||||||
|
let sync_snapshot = state
|
||||||
|
.site_sync
|
||||||
|
.as_ref()
|
||||||
|
.map(|w| w.snapshot_stats())
|
||||||
|
.unwrap_or_default();
|
||||||
|
let mut sync_errors: u64 = 0;
|
||||||
|
let mut last_sync_at: Option<f64> = None;
|
||||||
|
for s in sync_snapshot.values() {
|
||||||
|
sync_errors += s.errors;
|
||||||
|
if let Some(ts) = s.last_sync_at {
|
||||||
|
last_sync_at = match last_sync_at {
|
||||||
|
Some(prev) if prev > ts => Some(prev),
|
||||||
|
_ => Some(ts),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let now = chrono::Utc::now().timestamp_millis() as f64 / 1000.0;
|
||||||
|
serde_json::json!({
|
||||||
|
"site_id": local_site.as_ref().map(|s| s.site_id.clone()),
|
||||||
|
"display_name": local_site.as_ref().map(|s| s.display_name.clone()),
|
||||||
|
"endpoint": local_site.as_ref().map(|s| s.endpoint.clone()),
|
||||||
|
"region": local_site.as_ref().map(|s| s.region.clone()),
|
||||||
|
"priority": local_site.as_ref().map(|s| s.priority),
|
||||||
|
"capacity": {
|
||||||
|
"total_bytes": disk_total,
|
||||||
|
"available_bytes": disk_free,
|
||||||
|
},
|
||||||
|
"buckets": bucket_count,
|
||||||
|
"objects": total_objects,
|
||||||
|
"size_bytes": size_bytes,
|
||||||
|
"system": system,
|
||||||
|
"sync": {
|
||||||
|
"errors": sync_errors,
|
||||||
|
"last_sync_at": last_sync_at,
|
||||||
|
},
|
||||||
|
"generated_at": now,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get_cluster_overview(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Extension(principal): Extension<Principal>,
|
||||||
|
) -> Response {
|
||||||
|
if let Some(err) = require_admin_or_registered_peer(&state, &principal) {
|
||||||
|
return err;
|
||||||
|
}
|
||||||
|
{
|
||||||
|
let guard = state.cluster_overview_cache.lock();
|
||||||
|
if let Some((at, ref value)) = *guard {
|
||||||
|
if at.elapsed() < std::time::Duration::from_secs(10) {
|
||||||
|
return json_response(StatusCode::OK, value.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let value = build_cluster_overview(&state).await;
|
||||||
|
*state.cluster_overview_cache.lock() =
|
||||||
|
Some((std::time::Instant::now(), value.clone()));
|
||||||
|
json_response(StatusCode::OK, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get_sync_stats(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Extension(principal): Extension<Principal>,
|
||||||
|
) -> Response {
|
||||||
|
if let Some(err) = require_admin(&principal) {
|
||||||
|
return err;
|
||||||
|
}
|
||||||
|
let snapshot = match state.site_sync.as_ref() {
|
||||||
|
Some(worker) => worker.snapshot_stats(),
|
||||||
|
None => Default::default(),
|
||||||
|
};
|
||||||
|
let stats: Vec<serde_json::Value> = snapshot
|
||||||
|
.into_iter()
|
||||||
|
.map(|(bucket, s)| {
|
||||||
|
serde_json::json!({
|
||||||
|
"bucket": bucket,
|
||||||
|
"last_sync_at": s.last_sync_at,
|
||||||
|
"objects_pulled": s.objects_pulled,
|
||||||
|
"objects_skipped": s.objects_skipped,
|
||||||
|
"conflicts_resolved": s.conflicts_resolved,
|
||||||
|
"deletions_applied": s.deletions_applied,
|
||||||
|
"errors": s.errors,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
json_response(
|
||||||
|
StatusCode::OK,
|
||||||
|
serde_json::json!({
|
||||||
|
"enabled": state.site_sync.is_some(),
|
||||||
|
"stats": stats,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|||||||
@@ -20,6 +20,13 @@ fn xml_response(status: StatusCode, xml: String) -> Response {
|
|||||||
(status, [("content-type", "application/xml")], xml).into_response()
|
(status, [("content-type", "application/xml")], xml).into_response()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn stored_xml(value: &serde_json::Value) -> String {
|
||||||
|
match value {
|
||||||
|
serde_json::Value::String(s) => s.clone(),
|
||||||
|
other => other.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn storage_err(err: myfsio_storage::error::StorageError) -> Response {
|
fn storage_err(err: myfsio_storage::error::StorageError) -> Response {
|
||||||
let s3err = S3Error::from(err);
|
let s3err = S3Error::from(err);
|
||||||
let status =
|
let status =
|
||||||
@@ -52,17 +59,31 @@ fn custom_xml_error(status: StatusCode, code: &str, message: &str) -> Response {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_versioning(state: &AppState, bucket: &str) -> Response {
|
pub async fn get_versioning(state: &AppState, bucket: &str) -> Response {
|
||||||
match state.storage.is_versioning_enabled(bucket).await {
|
match state.storage.get_versioning_status(bucket).await {
|
||||||
Ok(enabled) => {
|
Ok(status) => {
|
||||||
let status_str = if enabled { "Enabled" } else { "Suspended" };
|
let body = match status {
|
||||||
let xml = format!(
|
myfsio_common::types::VersioningStatus::Enabled => {
|
||||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||||
<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
|
<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
|
||||||
<Status>{}</Status>\
|
<Status>Enabled</Status>\
|
||||||
</VersioningConfiguration>",
|
</VersioningConfiguration>"
|
||||||
status_str
|
.to_string()
|
||||||
);
|
}
|
||||||
xml_response(StatusCode::OK, xml)
|
myfsio_common::types::VersioningStatus::Suspended => {
|
||||||
|
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||||
|
<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
|
||||||
|
<Status>Suspended</Status>\
|
||||||
|
</VersioningConfiguration>"
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
myfsio_common::types::VersioningStatus::Disabled => {
|
||||||
|
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||||
|
<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
|
||||||
|
</VersioningConfiguration>"
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
xml_response(StatusCode::OK, body)
|
||||||
}
|
}
|
||||||
Err(e) => storage_err(e),
|
Err(e) => storage_err(e),
|
||||||
}
|
}
|
||||||
@@ -80,9 +101,22 @@ pub async fn put_versioning(state: &AppState, bucket: &str, body: Body) -> Respo
|
|||||||
};
|
};
|
||||||
|
|
||||||
let xml_str = String::from_utf8_lossy(&body_bytes);
|
let xml_str = String::from_utf8_lossy(&body_bytes);
|
||||||
let enabled = xml_str.contains("<Status>Enabled</Status>");
|
let status = if xml_str.contains("<Status>Enabled</Status>") {
|
||||||
|
myfsio_common::types::VersioningStatus::Enabled
|
||||||
|
} else if xml_str.contains("<Status>Suspended</Status>") {
|
||||||
|
myfsio_common::types::VersioningStatus::Suspended
|
||||||
|
} else {
|
||||||
|
return xml_response(
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
S3Error::new(
|
||||||
|
S3ErrorCode::MalformedXML,
|
||||||
|
"VersioningConfiguration Status must be Enabled or Suspended",
|
||||||
|
)
|
||||||
|
.to_xml(),
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
match state.storage.set_versioning(bucket, enabled).await {
|
match state.storage.set_versioning_status(bucket, status).await {
|
||||||
Ok(()) => StatusCode::OK.into_response(),
|
Ok(()) => StatusCode::OK.into_response(),
|
||||||
Err(e) => storage_err(e),
|
Err(e) => storage_err(e),
|
||||||
}
|
}
|
||||||
@@ -151,7 +185,7 @@ pub async fn get_cors(state: &AppState, bucket: &str) -> Response {
|
|||||||
match state.storage.get_bucket_config(bucket).await {
|
match state.storage.get_bucket_config(bucket).await {
|
||||||
Ok(config) => {
|
Ok(config) => {
|
||||||
if let Some(cors) = &config.cors {
|
if let Some(cors) = &config.cors {
|
||||||
xml_response(StatusCode::OK, cors.to_string())
|
xml_response(StatusCode::OK, stored_xml(cors))
|
||||||
} else {
|
} else {
|
||||||
xml_response(
|
xml_response(
|
||||||
StatusCode::NOT_FOUND,
|
StatusCode::NOT_FOUND,
|
||||||
@@ -214,15 +248,12 @@ pub async fn get_encryption(state: &AppState, bucket: &str) -> Response {
|
|||||||
match state.storage.get_bucket_config(bucket).await {
|
match state.storage.get_bucket_config(bucket).await {
|
||||||
Ok(config) => {
|
Ok(config) => {
|
||||||
if let Some(enc) = &config.encryption {
|
if let Some(enc) = &config.encryption {
|
||||||
xml_response(StatusCode::OK, enc.to_string())
|
xml_response(StatusCode::OK, stored_xml(enc))
|
||||||
} else {
|
} else {
|
||||||
xml_response(
|
xml_response(
|
||||||
StatusCode::NOT_FOUND,
|
StatusCode::NOT_FOUND,
|
||||||
S3Error::new(
|
S3Error::from_code(S3ErrorCode::ServerSideEncryptionConfigurationNotFoundError)
|
||||||
S3ErrorCode::InvalidRequest,
|
.to_xml(),
|
||||||
"The server side encryption configuration was not found",
|
|
||||||
)
|
|
||||||
.to_xml(),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -266,15 +297,11 @@ pub async fn get_lifecycle(state: &AppState, bucket: &str) -> Response {
|
|||||||
match state.storage.get_bucket_config(bucket).await {
|
match state.storage.get_bucket_config(bucket).await {
|
||||||
Ok(config) => {
|
Ok(config) => {
|
||||||
if let Some(lc) = &config.lifecycle {
|
if let Some(lc) = &config.lifecycle {
|
||||||
xml_response(StatusCode::OK, lc.to_string())
|
xml_response(StatusCode::OK, stored_xml(lc))
|
||||||
} else {
|
} else {
|
||||||
xml_response(
|
xml_response(
|
||||||
StatusCode::NOT_FOUND,
|
StatusCode::NOT_FOUND,
|
||||||
S3Error::new(
|
S3Error::from_code(S3ErrorCode::NoSuchLifecycleConfiguration).to_xml(),
|
||||||
S3ErrorCode::NoSuchKey,
|
|
||||||
"The lifecycle configuration does not exist",
|
|
||||||
)
|
|
||||||
.to_xml(),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -421,7 +448,7 @@ pub async fn get_policy(state: &AppState, bucket: &str) -> Response {
|
|||||||
} else {
|
} else {
|
||||||
xml_response(
|
xml_response(
|
||||||
StatusCode::NOT_FOUND,
|
StatusCode::NOT_FOUND,
|
||||||
S3Error::new(S3ErrorCode::NoSuchKey, "No bucket policy attached").to_xml(),
|
S3Error::from_code(S3ErrorCode::NoSuchBucketPolicy).to_xml(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -497,10 +524,7 @@ pub async fn get_replication(state: &AppState, bucket: &str) -> Response {
|
|||||||
match state.storage.get_bucket_config(bucket).await {
|
match state.storage.get_bucket_config(bucket).await {
|
||||||
Ok(config) => {
|
Ok(config) => {
|
||||||
if let Some(replication) = &config.replication {
|
if let Some(replication) = &config.replication {
|
||||||
match replication {
|
xml_response(StatusCode::OK, stored_xml(replication))
|
||||||
serde_json::Value::String(s) => xml_response(StatusCode::OK, s.clone()),
|
|
||||||
other => xml_response(StatusCode::OK, other.to_string()),
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
xml_response(
|
xml_response(
|
||||||
StatusCode::NOT_FOUND,
|
StatusCode::NOT_FOUND,
|
||||||
@@ -593,7 +617,7 @@ pub async fn get_acl(state: &AppState, bucket: &str) -> Response {
|
|||||||
match state.storage.get_bucket_config(bucket).await {
|
match state.storage.get_bucket_config(bucket).await {
|
||||||
Ok(config) => {
|
Ok(config) => {
|
||||||
if let Some(acl) = &config.acl {
|
if let Some(acl) = &config.acl {
|
||||||
xml_response(StatusCode::OK, acl.to_string())
|
xml_response(StatusCode::OK, stored_xml(acl))
|
||||||
} else {
|
} else {
|
||||||
let xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
let xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||||
<AccessControlPolicy xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
|
<AccessControlPolicy xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
|
||||||
@@ -633,7 +657,7 @@ pub async fn get_website(state: &AppState, bucket: &str) -> Response {
|
|||||||
match state.storage.get_bucket_config(bucket).await {
|
match state.storage.get_bucket_config(bucket).await {
|
||||||
Ok(config) => {
|
Ok(config) => {
|
||||||
if let Some(ws) = &config.website {
|
if let Some(ws) = &config.website {
|
||||||
xml_response(StatusCode::OK, ws.to_string())
|
xml_response(StatusCode::OK, stored_xml(ws))
|
||||||
} else {
|
} else {
|
||||||
xml_response(
|
xml_response(
|
||||||
StatusCode::NOT_FOUND,
|
StatusCode::NOT_FOUND,
|
||||||
@@ -685,7 +709,7 @@ pub async fn get_object_lock(state: &AppState, bucket: &str) -> Response {
|
|||||||
match state.storage.get_bucket_config(bucket).await {
|
match state.storage.get_bucket_config(bucket).await {
|
||||||
Ok(config) => {
|
Ok(config) => {
|
||||||
if let Some(ol) = &config.object_lock {
|
if let Some(ol) = &config.object_lock {
|
||||||
xml_response(StatusCode::OK, ol.to_string())
|
xml_response(StatusCode::OK, stored_xml(ol))
|
||||||
} else {
|
} else {
|
||||||
let xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
let xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||||
<ObjectLockConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
|
<ObjectLockConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
|
||||||
@@ -702,7 +726,7 @@ pub async fn get_notification(state: &AppState, bucket: &str) -> Response {
|
|||||||
match state.storage.get_bucket_config(bucket).await {
|
match state.storage.get_bucket_config(bucket).await {
|
||||||
Ok(config) => {
|
Ok(config) => {
|
||||||
if let Some(n) = &config.notification {
|
if let Some(n) = &config.notification {
|
||||||
xml_response(StatusCode::OK, n.to_string())
|
xml_response(StatusCode::OK, stored_xml(n))
|
||||||
} else {
|
} else {
|
||||||
let xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
let xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||||
<NotificationConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
|
<NotificationConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
|
||||||
@@ -1035,29 +1059,39 @@ pub async fn delete_logging(state: &AppState, bucket: &str) -> Response {
|
|||||||
|
|
||||||
fn s3_error_response(code: S3ErrorCode, message: &str, status: StatusCode) -> Response {
|
fn s3_error_response(code: S3ErrorCode, message: &str, status: StatusCode) -> Response {
|
||||||
let err = S3Error::new(code, message.to_string());
|
let err = S3Error::new(code, message.to_string());
|
||||||
(status, [("content-type", "application/xml")], err.to_xml()).into_response()
|
let code_str = code.as_str();
|
||||||
|
(
|
||||||
|
status,
|
||||||
|
[
|
||||||
|
("content-type", "application/xml"),
|
||||||
|
("x-amz-error-code", code_str),
|
||||||
|
],
|
||||||
|
err.to_xml(),
|
||||||
|
)
|
||||||
|
.into_response()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn list_object_versions(
|
pub async fn list_object_versions(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
prefix: Option<&str>,
|
prefix: Option<&str>,
|
||||||
|
delimiter: Option<&str>,
|
||||||
|
key_marker: Option<&str>,
|
||||||
|
version_id_marker: Option<&str>,
|
||||||
max_keys: usize,
|
max_keys: usize,
|
||||||
) -> Response {
|
) -> Response {
|
||||||
match state.storage.list_buckets().await {
|
match state.storage.bucket_exists(bucket).await {
|
||||||
Ok(buckets) => {
|
Ok(true) => {}
|
||||||
if !buckets.iter().any(|b| b.name == bucket) {
|
Ok(false) => {
|
||||||
return storage_err(myfsio_storage::error::StorageError::BucketNotFound(
|
return storage_err(myfsio_storage::error::StorageError::BucketNotFound(
|
||||||
bucket.to_string(),
|
bucket.to_string(),
|
||||||
));
|
));
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Err(e) => return storage_err(e),
|
Err(e) => return storage_err(e),
|
||||||
}
|
}
|
||||||
|
|
||||||
let fetch_limit = max_keys.saturating_add(1).max(1);
|
|
||||||
let params = myfsio_common::types::ListParams {
|
let params = myfsio_common::types::ListParams {
|
||||||
max_keys: fetch_limit,
|
max_keys: usize::MAX,
|
||||||
prefix: prefix.map(ToOwned::to_owned),
|
prefix: prefix.map(ToOwned::to_owned),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
@@ -1066,7 +1100,8 @@ pub async fn list_object_versions(
|
|||||||
Ok(result) => result,
|
Ok(result) => result,
|
||||||
Err(e) => return storage_err(e),
|
Err(e) => return storage_err(e),
|
||||||
};
|
};
|
||||||
let objects = object_result.objects;
|
let live_objects = object_result.objects;
|
||||||
|
|
||||||
let archived_versions = match state
|
let archived_versions = match state
|
||||||
.storage
|
.storage
|
||||||
.list_bucket_object_versions(bucket, prefix)
|
.list_bucket_object_versions(bucket, prefix)
|
||||||
@@ -1076,71 +1111,232 @@ pub async fn list_object_versions(
|
|||||||
Err(e) => return storage_err(e),
|
Err(e) => return storage_err(e),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct Entry {
|
||||||
|
key: String,
|
||||||
|
version_id: String,
|
||||||
|
last_modified: chrono::DateTime<chrono::Utc>,
|
||||||
|
etag: Option<String>,
|
||||||
|
size: u64,
|
||||||
|
storage_class: String,
|
||||||
|
is_delete_marker: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut entries: Vec<Entry> = Vec::with_capacity(live_objects.len() + archived_versions.len());
|
||||||
|
for obj in &live_objects {
|
||||||
|
entries.push(Entry {
|
||||||
|
key: obj.key.clone(),
|
||||||
|
version_id: obj.version_id.clone().unwrap_or_else(|| "null".to_string()),
|
||||||
|
last_modified: obj.last_modified,
|
||||||
|
etag: obj.etag.clone(),
|
||||||
|
size: obj.size,
|
||||||
|
storage_class: obj
|
||||||
|
.storage_class
|
||||||
|
.clone()
|
||||||
|
.unwrap_or_else(|| "STANDARD".to_string()),
|
||||||
|
is_delete_marker: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
for version in &archived_versions {
|
||||||
|
entries.push(Entry {
|
||||||
|
key: version.key.clone(),
|
||||||
|
version_id: version.version_id.clone(),
|
||||||
|
last_modified: version.last_modified,
|
||||||
|
etag: version.etag.clone(),
|
||||||
|
size: version.size,
|
||||||
|
storage_class: "STANDARD".to_string(),
|
||||||
|
is_delete_marker: version.is_delete_marker,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
entries.sort_by(|a, b| {
|
||||||
|
a.key
|
||||||
|
.cmp(&b.key)
|
||||||
|
.then_with(|| b.last_modified.cmp(&a.last_modified))
|
||||||
|
.then_with(|| a.version_id.cmp(&b.version_id))
|
||||||
|
});
|
||||||
|
|
||||||
|
let mut latest_marked: std::collections::HashSet<String> = std::collections::HashSet::new();
|
||||||
|
let mut is_latest_flags: Vec<bool> = Vec::with_capacity(entries.len());
|
||||||
|
for entry in &entries {
|
||||||
|
if latest_marked.insert(entry.key.clone()) {
|
||||||
|
is_latest_flags.push(true);
|
||||||
|
} else {
|
||||||
|
is_latest_flags.push(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let km = key_marker.unwrap_or("");
|
||||||
|
let vim = version_id_marker.unwrap_or("");
|
||||||
|
let start_index = if km.is_empty() {
|
||||||
|
0
|
||||||
|
} else if vim.is_empty() {
|
||||||
|
entries
|
||||||
|
.iter()
|
||||||
|
.position(|e| e.key.as_str() > km)
|
||||||
|
.unwrap_or(entries.len())
|
||||||
|
} else if let Some(pos) = entries
|
||||||
|
.iter()
|
||||||
|
.position(|e| e.key == km && e.version_id == vim)
|
||||||
|
{
|
||||||
|
pos + 1
|
||||||
|
} else {
|
||||||
|
entries
|
||||||
|
.iter()
|
||||||
|
.position(|e| e.key.as_str() > km)
|
||||||
|
.unwrap_or(entries.len())
|
||||||
|
};
|
||||||
|
|
||||||
|
let delim = delimiter.unwrap_or("");
|
||||||
|
let prefix_str = prefix.unwrap_or("");
|
||||||
|
|
||||||
|
let mut common_prefixes: Vec<String> = Vec::new();
|
||||||
|
let mut seen_prefixes: std::collections::HashSet<String> = std::collections::HashSet::new();
|
||||||
|
let mut rendered = String::new();
|
||||||
|
let mut count = 0usize;
|
||||||
|
let mut is_truncated = false;
|
||||||
|
let mut next_key_marker: Option<String> = None;
|
||||||
|
let mut next_version_id_marker: Option<String> = None;
|
||||||
|
let mut last_emitted: Option<(String, String)> = None;
|
||||||
|
|
||||||
|
let mut idx = start_index;
|
||||||
|
while idx < entries.len() {
|
||||||
|
let entry = &entries[idx];
|
||||||
|
let is_latest = is_latest_flags[idx];
|
||||||
|
|
||||||
|
if !delim.is_empty() {
|
||||||
|
let rest = entry.key.strip_prefix(prefix_str).unwrap_or(&entry.key);
|
||||||
|
if let Some(delim_pos) = rest.find(delim) {
|
||||||
|
let grouped = entry.key[..prefix_str.len() + delim_pos + delim.len()].to_string();
|
||||||
|
if seen_prefixes.contains(&grouped) {
|
||||||
|
idx += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if count >= max_keys {
|
||||||
|
is_truncated = true;
|
||||||
|
if let Some((k, v)) = last_emitted.clone() {
|
||||||
|
next_key_marker = Some(k);
|
||||||
|
next_version_id_marker = Some(v);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
common_prefixes.push(grouped.clone());
|
||||||
|
seen_prefixes.insert(grouped.clone());
|
||||||
|
count += 1;
|
||||||
|
|
||||||
|
let mut group_last = (entry.key.clone(), entry.version_id.clone());
|
||||||
|
idx += 1;
|
||||||
|
while idx < entries.len() && entries[idx].key.starts_with(&grouped) {
|
||||||
|
group_last = (entries[idx].key.clone(), entries[idx].version_id.clone());
|
||||||
|
idx += 1;
|
||||||
|
}
|
||||||
|
last_emitted = Some(group_last);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if count >= max_keys {
|
||||||
|
is_truncated = true;
|
||||||
|
if let Some((k, v)) = last_emitted.clone() {
|
||||||
|
next_key_marker = Some(k);
|
||||||
|
next_version_id_marker = Some(v);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
let tag = if entry.is_delete_marker {
|
||||||
|
"DeleteMarker"
|
||||||
|
} else {
|
||||||
|
"Version"
|
||||||
|
};
|
||||||
|
rendered.push_str(&format!("<{}>", tag));
|
||||||
|
rendered.push_str(&format!("<Key>{}</Key>", xml_escape(&entry.key)));
|
||||||
|
rendered.push_str(&format!(
|
||||||
|
"<VersionId>{}</VersionId>",
|
||||||
|
xml_escape(&entry.version_id)
|
||||||
|
));
|
||||||
|
rendered.push_str(&format!("<IsLatest>{}</IsLatest>", is_latest));
|
||||||
|
rendered.push_str(&format!(
|
||||||
|
"<LastModified>{}</LastModified>",
|
||||||
|
myfsio_xml::response::format_s3_datetime(&entry.last_modified)
|
||||||
|
));
|
||||||
|
if !entry.is_delete_marker {
|
||||||
|
if let Some(ref etag) = entry.etag {
|
||||||
|
rendered.push_str(&format!("<ETag>\"{}\"</ETag>", xml_escape(etag)));
|
||||||
|
}
|
||||||
|
rendered.push_str(&format!("<Size>{}</Size>", entry.size));
|
||||||
|
rendered.push_str(&format!(
|
||||||
|
"<StorageClass>{}</StorageClass>",
|
||||||
|
xml_escape(&entry.storage_class)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
rendered.push_str(&format!("</{}>", tag));
|
||||||
|
|
||||||
|
last_emitted = Some((entry.key.clone(), entry.version_id.clone()));
|
||||||
|
count += 1;
|
||||||
|
idx += 1;
|
||||||
|
}
|
||||||
|
|
||||||
let mut xml = String::from(
|
let mut xml = String::from(
|
||||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||||
<ListVersionsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">",
|
<ListVersionsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">",
|
||||||
);
|
);
|
||||||
xml.push_str(&format!("<Name>{}</Name>", xml_escape(bucket)));
|
xml.push_str(&format!("<Name>{}</Name>", xml_escape(bucket)));
|
||||||
xml.push_str(&format!(
|
xml.push_str(&format!("<Prefix>{}</Prefix>", xml_escape(prefix_str)));
|
||||||
"<Prefix>{}</Prefix>",
|
if !km.is_empty() {
|
||||||
xml_escape(prefix.unwrap_or(""))
|
xml.push_str(&format!("<KeyMarker>{}</KeyMarker>", xml_escape(km)));
|
||||||
));
|
} else {
|
||||||
|
xml.push_str("<KeyMarker></KeyMarker>");
|
||||||
|
}
|
||||||
|
if !vim.is_empty() {
|
||||||
|
xml.push_str(&format!(
|
||||||
|
"<VersionIdMarker>{}</VersionIdMarker>",
|
||||||
|
xml_escape(vim)
|
||||||
|
));
|
||||||
|
} else {
|
||||||
|
xml.push_str("<VersionIdMarker></VersionIdMarker>");
|
||||||
|
}
|
||||||
xml.push_str(&format!("<MaxKeys>{}</MaxKeys>", max_keys));
|
xml.push_str(&format!("<MaxKeys>{}</MaxKeys>", max_keys));
|
||||||
|
if !delim.is_empty() {
|
||||||
let current_count = objects.len().min(max_keys);
|
xml.push_str(&format!("<Delimiter>{}</Delimiter>", xml_escape(delim)));
|
||||||
let remaining = max_keys.saturating_sub(current_count);
|
}
|
||||||
let archived_count = archived_versions.len().min(remaining);
|
|
||||||
let is_truncated = object_result.is_truncated
|
|
||||||
|| objects.len() > current_count
|
|
||||||
|| archived_versions.len() > archived_count;
|
|
||||||
xml.push_str(&format!("<IsTruncated>{}</IsTruncated>", is_truncated));
|
xml.push_str(&format!("<IsTruncated>{}</IsTruncated>", is_truncated));
|
||||||
|
if let Some(ref nk) = next_key_marker {
|
||||||
for obj in objects.iter().take(current_count) {
|
|
||||||
xml.push_str("<Version>");
|
|
||||||
xml.push_str(&format!("<Key>{}</Key>", xml_escape(&obj.key)));
|
|
||||||
xml.push_str("<VersionId>null</VersionId>");
|
|
||||||
xml.push_str("<IsLatest>true</IsLatest>");
|
|
||||||
xml.push_str(&format!(
|
xml.push_str(&format!(
|
||||||
"<LastModified>{}</LastModified>",
|
"<NextKeyMarker>{}</NextKeyMarker>",
|
||||||
myfsio_xml::response::format_s3_datetime(&obj.last_modified)
|
xml_escape(nk)
|
||||||
));
|
));
|
||||||
if let Some(ref etag) = obj.etag {
|
}
|
||||||
xml.push_str(&format!("<ETag>\"{}\"</ETag>", xml_escape(etag)));
|
if let Some(ref nv) = next_version_id_marker {
|
||||||
}
|
|
||||||
xml.push_str(&format!("<Size>{}</Size>", obj.size));
|
|
||||||
xml.push_str(&format!(
|
xml.push_str(&format!(
|
||||||
"<StorageClass>{}</StorageClass>",
|
"<NextVersionIdMarker>{}</NextVersionIdMarker>",
|
||||||
xml_escape(obj.storage_class.as_deref().unwrap_or("STANDARD"))
|
xml_escape(nv)
|
||||||
));
|
));
|
||||||
xml.push_str("</Version>");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for version in archived_versions.iter().take(archived_count) {
|
xml.push_str(&rendered);
|
||||||
xml.push_str("<Version>");
|
for cp in &common_prefixes {
|
||||||
xml.push_str(&format!("<Key>{}</Key>", xml_escape(&version.key)));
|
|
||||||
xml.push_str(&format!(
|
xml.push_str(&format!(
|
||||||
"<VersionId>{}</VersionId>",
|
"<CommonPrefixes><Prefix>{}</Prefix></CommonPrefixes>",
|
||||||
xml_escape(&version.version_id)
|
xml_escape(cp)
|
||||||
));
|
));
|
||||||
xml.push_str("<IsLatest>false</IsLatest>");
|
|
||||||
xml.push_str(&format!(
|
|
||||||
"<LastModified>{}</LastModified>",
|
|
||||||
myfsio_xml::response::format_s3_datetime(&version.last_modified)
|
|
||||||
));
|
|
||||||
if let Some(ref etag) = version.etag {
|
|
||||||
xml.push_str(&format!("<ETag>\"{}\"</ETag>", xml_escape(etag)));
|
|
||||||
}
|
|
||||||
xml.push_str(&format!("<Size>{}</Size>", version.size));
|
|
||||||
xml.push_str("<StorageClass>STANDARD</StorageClass>");
|
|
||||||
xml.push_str("</Version>");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
xml.push_str("</ListVersionsResult>");
|
xml.push_str("</ListVersionsResult>");
|
||||||
xml_response(StatusCode::OK, xml)
|
xml_response(StatusCode::OK, xml)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_object_tagging(state: &AppState, bucket: &str, key: &str) -> Response {
|
pub async fn get_object_tagging(
|
||||||
match state.storage.get_object_tags(bucket, key).await {
|
state: &AppState,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: Option<&str>,
|
||||||
|
) -> Response {
|
||||||
|
let lookup = match version_id {
|
||||||
|
Some(v) => state.storage.get_object_version_tags(bucket, key, v).await,
|
||||||
|
None => state.storage.get_object_tags(bucket, key).await,
|
||||||
|
};
|
||||||
|
match lookup {
|
||||||
Ok(tags) => {
|
Ok(tags) => {
|
||||||
let mut xml = String::from(
|
let mut xml = String::from(
|
||||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||||
@@ -1182,6 +1378,26 @@ pub async fn put_object_tagging(state: &AppState, bucket: &str, key: &str, body:
|
|||||||
.to_xml(),
|
.to_xml(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
for tag in &tags {
|
||||||
|
if tag.key.is_empty() || tag.key.len() > 128 {
|
||||||
|
return xml_response(
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
S3Error::new(S3ErrorCode::InvalidTag, "Tag key length must be 1-128").to_xml(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if tag.value.len() > 256 {
|
||||||
|
return xml_response(
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
S3Error::new(S3ErrorCode::InvalidTag, "Tag value length must be 0-256").to_xml(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if tag.key.contains('=') {
|
||||||
|
return xml_response(
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
S3Error::new(S3ErrorCode::InvalidTag, "Tag keys must not contain '='").to_xml(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
match state.storage.set_object_tags(bucket, key, &tags).await {
|
match state.storage.set_object_tags(bucket, key, &tags).await {
|
||||||
Ok(()) => StatusCode::OK.into_response(),
|
Ok(()) => StatusCode::OK.into_response(),
|
||||||
|
|||||||
@@ -14,17 +14,32 @@ use crate::state::AppState;
|
|||||||
fn json_ok(value: Value) -> Response {
|
fn json_ok(value: Value) -> Response {
|
||||||
(
|
(
|
||||||
StatusCode::OK,
|
StatusCode::OK,
|
||||||
[("content-type", "application/json")],
|
[("content-type", "application/x-amz-json-1.1")],
|
||||||
value.to_string(),
|
value.to_string(),
|
||||||
)
|
)
|
||||||
.into_response()
|
.into_response()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn json_err(status: StatusCode, msg: &str) -> Response {
|
fn json_err(status: StatusCode, msg: &str) -> Response {
|
||||||
|
let type_name = match status {
|
||||||
|
StatusCode::BAD_REQUEST => "ValidationException",
|
||||||
|
StatusCode::NOT_FOUND => "NotFoundException",
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE => "KMSInternalException",
|
||||||
|
StatusCode::FORBIDDEN => "AccessDeniedException",
|
||||||
|
_ => "KMSInternalException",
|
||||||
|
};
|
||||||
|
json_err_typed(status, type_name, msg)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn json_err_typed(status: StatusCode, type_name: &str, msg: &str) -> Response {
|
||||||
(
|
(
|
||||||
status,
|
status,
|
||||||
[("content-type", "application/json")],
|
[("content-type", "application/x-amz-json-1.1")],
|
||||||
json!({"error": msg}).to_string(),
|
json!({
|
||||||
|
"__type": format!("com.amazonaws.kms#{}", type_name),
|
||||||
|
"message": msg,
|
||||||
|
})
|
||||||
|
.to_string(),
|
||||||
)
|
)
|
||||||
.into_response()
|
.into_response()
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -511,11 +511,20 @@ fn s3_error_response(err: S3Error) -> Response {
|
|||||||
} else {
|
} else {
|
||||||
err.resource.clone()
|
err.resource.clone()
|
||||||
};
|
};
|
||||||
|
let code_str = err.code.as_str();
|
||||||
let body = err
|
let body = err
|
||||||
.with_resource(resource)
|
.with_resource(resource)
|
||||||
.with_request_id(uuid::Uuid::new_v4().simple().to_string())
|
.with_request_id(uuid::Uuid::new_v4().simple().to_string())
|
||||||
.to_xml();
|
.to_xml();
|
||||||
(status, [("content-type", "application/xml")], body).into_response()
|
(
|
||||||
|
status,
|
||||||
|
[
|
||||||
|
("content-type", "application/xml"),
|
||||||
|
("x-amz-error-code", code_str),
|
||||||
|
],
|
||||||
|
body,
|
||||||
|
)
|
||||||
|
.into_response()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_stats_xml(bytes_scanned: usize, bytes_returned: usize) -> String {
|
fn build_stats_xml(bytes_scanned: usize, bytes_returned: usize) -> String {
|
||||||
|
|||||||
86
crates/myfsio-server/src/handlers/static_assets.rs
Normal file
86
crates/myfsio-server/src/handlers/static_assets.rs
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
use axum::{
|
||||||
|
body::Body,
|
||||||
|
extract::{Path, State},
|
||||||
|
http::{header, HeaderMap, HeaderValue, StatusCode},
|
||||||
|
response::{IntoResponse, Response},
|
||||||
|
};
|
||||||
|
|
||||||
|
use crate::embedded;
|
||||||
|
use crate::state::AppState;
|
||||||
|
|
||||||
|
pub async fn serve(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Path(path): Path<String>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
) -> Response {
|
||||||
|
let normalized = path.trim_start_matches('/').to_string();
|
||||||
|
if normalized.is_empty() || normalized.contains("..") {
|
||||||
|
return StatusCode::NOT_FOUND.into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let use_disk = std::env::var("STATIC_DIR").is_ok() && state.config.static_dir.is_dir();
|
||||||
|
if use_disk {
|
||||||
|
let candidate = state.config.static_dir.join(&normalized);
|
||||||
|
if let Ok(canonical) = candidate.canonicalize() {
|
||||||
|
if canonical.starts_with(
|
||||||
|
state
|
||||||
|
.config
|
||||||
|
.static_dir
|
||||||
|
.canonicalize()
|
||||||
|
.unwrap_or_else(|_| state.config.static_dir.clone()),
|
||||||
|
) {
|
||||||
|
if let Ok(bytes) = tokio::fs::read(&canonical).await {
|
||||||
|
let mime = mime_guess::from_path(&canonical).first_or_octet_stream();
|
||||||
|
return build_response(&normalized, bytes, mime.as_ref(), &headers);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return StatusCode::NOT_FOUND.into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
match embedded::static_file(&normalized) {
|
||||||
|
Some(file) => {
|
||||||
|
let mime = mime_guess::from_path(&normalized).first_or_octet_stream();
|
||||||
|
build_response(&normalized, file.data.into_owned(), mime.as_ref(), &headers)
|
||||||
|
}
|
||||||
|
None => StatusCode::NOT_FOUND.into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_response(_path: &str, bytes: Vec<u8>, mime: &str, request_headers: &HeaderMap) -> Response {
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
hasher.update(&bytes);
|
||||||
|
let etag = format!("\"{:.16x}\"", hasher.finalize());
|
||||||
|
|
||||||
|
if let Some(if_none_match) = request_headers
|
||||||
|
.get(header::IF_NONE_MATCH)
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
{
|
||||||
|
if if_none_match.split(',').any(|tag| tag.trim() == etag) {
|
||||||
|
let mut resp = Response::new(Body::empty());
|
||||||
|
*resp.status_mut() = StatusCode::NOT_MODIFIED;
|
||||||
|
if let Ok(v) = HeaderValue::from_str(&etag) {
|
||||||
|
resp.headers_mut().insert(header::ETAG, v);
|
||||||
|
}
|
||||||
|
return resp;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let len = bytes.len();
|
||||||
|
let mut response = Response::new(Body::from(bytes));
|
||||||
|
if let Ok(value) = HeaderValue::from_str(mime) {
|
||||||
|
response.headers_mut().insert(header::CONTENT_TYPE, value);
|
||||||
|
}
|
||||||
|
response
|
||||||
|
.headers_mut()
|
||||||
|
.insert(header::CONTENT_LENGTH, HeaderValue::from(len));
|
||||||
|
if let Ok(v) = HeaderValue::from_str(&etag) {
|
||||||
|
response.headers_mut().insert(header::ETAG, v);
|
||||||
|
}
|
||||||
|
response.headers_mut().insert(
|
||||||
|
header::CACHE_CONTROL,
|
||||||
|
HeaderValue::from_static("public, max-age=300, must-revalidate"),
|
||||||
|
);
|
||||||
|
response
|
||||||
|
}
|
||||||
@@ -56,6 +56,7 @@ pub async fn login_submit(
|
|||||||
})
|
})
|
||||||
.unwrap_or_else(|| access_key.to_string());
|
.unwrap_or_else(|| access_key.to_string());
|
||||||
|
|
||||||
|
session.rotate_id();
|
||||||
session.write(|s| {
|
session.write(|s| {
|
||||||
s.user_id = Some(access_key.to_string());
|
s.user_id = Some(access_key.to_string());
|
||||||
s.display_name = Some(display);
|
s.display_name = Some(display);
|
||||||
@@ -117,16 +118,6 @@ pub async fn logout(Extension(session): Extension<SessionHandle>) -> Response {
|
|||||||
Redirect::to("/login").into_response()
|
Redirect::to("/login").into_response()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn csrf_error_page(
|
|
||||||
State(state): State<AppState>,
|
|
||||||
Extension(session): Extension<SessionHandle>,
|
|
||||||
) -> Response {
|
|
||||||
let ctx = base_context(&session, None);
|
|
||||||
let mut resp = render(&state, "csrf_error.html", &ctx);
|
|
||||||
*resp.status_mut() = StatusCode::FORBIDDEN;
|
|
||||||
resp
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn root_redirect() -> Response {
|
pub async fn root_redirect() -> Response {
|
||||||
Redirect::to("/ui/buckets").into_response()
|
Redirect::to("/ui/buckets").into_response()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -121,12 +121,15 @@ fn storage_status(err: &StorageError) -> StatusCode {
|
|||||||
| StorageError::ObjectNotFound { .. }
|
| StorageError::ObjectNotFound { .. }
|
||||||
| StorageError::VersionNotFound { .. }
|
| StorageError::VersionNotFound { .. }
|
||||||
| StorageError::UploadNotFound(_) => StatusCode::NOT_FOUND,
|
| StorageError::UploadNotFound(_) => StatusCode::NOT_FOUND,
|
||||||
|
StorageError::DeleteMarker { .. } => StatusCode::NOT_FOUND,
|
||||||
|
StorageError::MethodNotAllowed(_) => StatusCode::METHOD_NOT_ALLOWED,
|
||||||
StorageError::InvalidBucketName(_)
|
StorageError::InvalidBucketName(_)
|
||||||
| StorageError::InvalidObjectKey(_)
|
| StorageError::InvalidObjectKey(_)
|
||||||
| StorageError::InvalidRange
|
| StorageError::InvalidRange
|
||||||
| StorageError::QuotaExceeded(_) => StatusCode::BAD_REQUEST,
|
| StorageError::QuotaExceeded(_) => StatusCode::BAD_REQUEST,
|
||||||
StorageError::BucketAlreadyExists(_) => StatusCode::CONFLICT,
|
StorageError::BucketAlreadyExists(_) => StatusCode::CONFLICT,
|
||||||
StorageError::BucketNotEmpty(_) => StatusCode::CONFLICT,
|
StorageError::BucketNotEmpty(_) => StatusCode::CONFLICT,
|
||||||
|
StorageError::ObjectCorrupted { .. } => StatusCode::UNPROCESSABLE_ENTITY,
|
||||||
StorageError::Io(_) | StorageError::Json(_) | StorageError::Internal(_) => {
|
StorageError::Io(_) | StorageError::Json(_) | StorageError::Internal(_) => {
|
||||||
StatusCode::INTERNAL_SERVER_ERROR
|
StatusCode::INTERNAL_SERVER_ERROR
|
||||||
}
|
}
|
||||||
@@ -904,6 +907,35 @@ pub struct ListObjectsQuery {
|
|||||||
pub prefix: Option<String>,
|
pub prefix: Option<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub start_after: Option<String>,
|
pub start_after: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub delimiter: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn object_json(bucket_name: &str, o: &myfsio_common::types::ObjectMeta) -> Value {
|
||||||
|
json!({
|
||||||
|
"key": o.key,
|
||||||
|
"size": o.size,
|
||||||
|
"last_modified": o.last_modified.to_rfc3339(),
|
||||||
|
"last_modified_iso": o.last_modified.to_rfc3339(),
|
||||||
|
"last_modified_display": o.last_modified.format("%Y-%m-%d %H:%M:%S").to_string(),
|
||||||
|
"etag": o.etag.clone().unwrap_or_default(),
|
||||||
|
"storage_class": o.storage_class.clone().unwrap_or_else(|| "STANDARD".to_string()),
|
||||||
|
"content_type": o.content_type.clone().unwrap_or_default(),
|
||||||
|
"download_url": build_ui_object_url(bucket_name, &o.key, "download"),
|
||||||
|
"preview_url": build_ui_object_url(bucket_name, &o.key, "preview"),
|
||||||
|
"delete_endpoint": build_ui_object_url(bucket_name, &o.key, "delete"),
|
||||||
|
"presign_endpoint": build_ui_object_url(bucket_name, &o.key, "presign"),
|
||||||
|
"metadata_url": build_ui_object_url(bucket_name, &o.key, "metadata"),
|
||||||
|
"versions_endpoint": build_ui_object_url(bucket_name, &o.key, "versions"),
|
||||||
|
"restore_template": format!(
|
||||||
|
"/ui/buckets/{}/objects/{}/restore/VERSION_ID_PLACEHOLDER",
|
||||||
|
bucket_name,
|
||||||
|
encode_object_key(&o.key)
|
||||||
|
),
|
||||||
|
"tags_url": build_ui_object_url(bucket_name, &o.key, "tags"),
|
||||||
|
"copy_url": build_ui_object_url(bucket_name, &o.key, "copy"),
|
||||||
|
"move_url": build_ui_object_url(bucket_name, &o.key, "move"),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn list_bucket_objects(
|
pub async fn list_bucket_objects(
|
||||||
@@ -917,6 +949,49 @@ pub async fn list_bucket_objects(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let max_keys = q.max_keys.unwrap_or(1000).min(5000);
|
let max_keys = q.max_keys.unwrap_or(1000).min(5000);
|
||||||
|
let versioning_enabled = state
|
||||||
|
.storage
|
||||||
|
.is_versioning_enabled(&bucket_name)
|
||||||
|
.await
|
||||||
|
.unwrap_or(false);
|
||||||
|
let stats = state.storage.bucket_stats(&bucket_name).await.ok();
|
||||||
|
let total_count = stats.as_ref().map(|s| s.objects).unwrap_or(0);
|
||||||
|
|
||||||
|
let use_shallow = q.delimiter.as_deref() == Some("/");
|
||||||
|
|
||||||
|
if use_shallow {
|
||||||
|
let params = myfsio_common::types::ShallowListParams {
|
||||||
|
prefix: q.prefix.clone().unwrap_or_default(),
|
||||||
|
delimiter: "/".to_string(),
|
||||||
|
max_keys,
|
||||||
|
continuation_token: q.continuation_token.clone(),
|
||||||
|
};
|
||||||
|
return match state
|
||||||
|
.storage
|
||||||
|
.list_objects_shallow(&bucket_name, ¶ms)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(res) => {
|
||||||
|
let objects: Vec<Value> = res
|
||||||
|
.objects
|
||||||
|
.iter()
|
||||||
|
.map(|o| object_json(&bucket_name, o))
|
||||||
|
.collect();
|
||||||
|
Json(json!({
|
||||||
|
"versioning_enabled": versioning_enabled,
|
||||||
|
"total_count": total_count,
|
||||||
|
"is_truncated": res.is_truncated,
|
||||||
|
"next_continuation_token": res.next_continuation_token,
|
||||||
|
"url_templates": url_templates_for(&bucket_name),
|
||||||
|
"objects": objects,
|
||||||
|
"common_prefixes": res.common_prefixes,
|
||||||
|
}))
|
||||||
|
.into_response()
|
||||||
|
}
|
||||||
|
Err(e) => storage_json_error(e),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
let params = ListParams {
|
let params = ListParams {
|
||||||
max_keys,
|
max_keys,
|
||||||
continuation_token: q.continuation_token.clone(),
|
continuation_token: q.continuation_token.clone(),
|
||||||
@@ -924,46 +999,12 @@ pub async fn list_bucket_objects(
|
|||||||
start_after: q.start_after.clone(),
|
start_after: q.start_after.clone(),
|
||||||
};
|
};
|
||||||
|
|
||||||
let versioning_enabled = state
|
|
||||||
.storage
|
|
||||||
.is_versioning_enabled(&bucket_name)
|
|
||||||
.await
|
|
||||||
.unwrap_or(false);
|
|
||||||
|
|
||||||
let stats = state.storage.bucket_stats(&bucket_name).await.ok();
|
|
||||||
let total_count = stats.as_ref().map(|s| s.objects).unwrap_or(0);
|
|
||||||
|
|
||||||
match state.storage.list_objects(&bucket_name, ¶ms).await {
|
match state.storage.list_objects(&bucket_name, ¶ms).await {
|
||||||
Ok(res) => {
|
Ok(res) => {
|
||||||
let objects: Vec<Value> = res
|
let objects: Vec<Value> = res
|
||||||
.objects
|
.objects
|
||||||
.iter()
|
.iter()
|
||||||
.map(|o| {
|
.map(|o| object_json(&bucket_name, o))
|
||||||
json!({
|
|
||||||
"key": o.key,
|
|
||||||
"size": o.size,
|
|
||||||
"last_modified": o.last_modified.to_rfc3339(),
|
|
||||||
"last_modified_iso": o.last_modified.to_rfc3339(),
|
|
||||||
"last_modified_display": o.last_modified.format("%Y-%m-%d %H:%M:%S").to_string(),
|
|
||||||
"etag": o.etag.clone().unwrap_or_default(),
|
|
||||||
"storage_class": o.storage_class.clone().unwrap_or_else(|| "STANDARD".to_string()),
|
|
||||||
"content_type": o.content_type.clone().unwrap_or_default(),
|
|
||||||
"download_url": build_ui_object_url(&bucket_name, &o.key, "download"),
|
|
||||||
"preview_url": build_ui_object_url(&bucket_name, &o.key, "preview"),
|
|
||||||
"delete_endpoint": build_ui_object_url(&bucket_name, &o.key, "delete"),
|
|
||||||
"presign_endpoint": build_ui_object_url(&bucket_name, &o.key, "presign"),
|
|
||||||
"metadata_url": build_ui_object_url(&bucket_name, &o.key, "metadata"),
|
|
||||||
"versions_endpoint": build_ui_object_url(&bucket_name, &o.key, "versions"),
|
|
||||||
"restore_template": format!(
|
|
||||||
"/ui/buckets/{}/objects/{}/restore/VERSION_ID_PLACEHOLDER",
|
|
||||||
bucket_name,
|
|
||||||
encode_object_key(&o.key)
|
|
||||||
),
|
|
||||||
"tags_url": build_ui_object_url(&bucket_name, &o.key, "tags"),
|
|
||||||
"copy_url": build_ui_object_url(&bucket_name, &o.key, "copy"),
|
|
||||||
"move_url": build_ui_object_url(&bucket_name, &o.key, "move"),
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
Json(json!({
|
Json(json!({
|
||||||
@@ -1006,41 +1047,62 @@ pub async fn stream_bucket_objects(
|
|||||||
let stats = state.storage.bucket_stats(&bucket_name).await.ok();
|
let stats = state.storage.bucket_stats(&bucket_name).await.ok();
|
||||||
let total_count = stats.as_ref().map(|s| s.objects).unwrap_or(0);
|
let total_count = stats.as_ref().map(|s| s.objects).unwrap_or(0);
|
||||||
|
|
||||||
let mut lines: Vec<String> = Vec::new();
|
|
||||||
lines.push(
|
|
||||||
json!({
|
|
||||||
"type": "meta",
|
|
||||||
"url_templates": url_templates_for(&bucket_name),
|
|
||||||
"versioning_enabled": versioning_enabled,
|
|
||||||
})
|
|
||||||
.to_string(),
|
|
||||||
);
|
|
||||||
lines.push(json!({ "type": "count", "total_count": total_count }).to_string());
|
|
||||||
|
|
||||||
let use_delimiter = q.delimiter.as_deref() == Some("/");
|
let use_delimiter = q.delimiter.as_deref() == Some("/");
|
||||||
let prefix = q.prefix.clone().unwrap_or_default();
|
let prefix = q.prefix.clone().unwrap_or_default();
|
||||||
|
|
||||||
if use_delimiter {
|
let (tx, rx) = tokio::sync::mpsc::channel::<Result<bytes::Bytes, std::io::Error>>(64);
|
||||||
let mut token: Option<String> = None;
|
|
||||||
loop {
|
let meta_line = json!({
|
||||||
let params = myfsio_common::types::ShallowListParams {
|
"type": "meta",
|
||||||
prefix: prefix.clone(),
|
"url_templates": url_templates_for(&bucket_name),
|
||||||
delimiter: "/".to_string(),
|
"versioning_enabled": versioning_enabled,
|
||||||
max_keys: UI_OBJECT_BROWSER_MAX_KEYS,
|
})
|
||||||
continuation_token: token.clone(),
|
.to_string()
|
||||||
};
|
+ "\n";
|
||||||
match state
|
let count_line = json!({ "type": "count", "total_count": total_count }).to_string() + "\n";
|
||||||
.storage
|
|
||||||
.list_objects_shallow(&bucket_name, ¶ms)
|
let storage = state.storage.clone();
|
||||||
.await
|
let bucket = bucket_name.clone();
|
||||||
{
|
|
||||||
Ok(res) => {
|
tokio::spawn(async move {
|
||||||
for p in &res.common_prefixes {
|
if tx
|
||||||
lines.push(json!({ "type": "folder", "prefix": p }).to_string());
|
.send(Ok(bytes::Bytes::from(meta_line.into_bytes())))
|
||||||
}
|
.await
|
||||||
for o in &res.objects {
|
.is_err()
|
||||||
lines.push(
|
{
|
||||||
json!({
|
return;
|
||||||
|
}
|
||||||
|
if tx
|
||||||
|
.send(Ok(bytes::Bytes::from(count_line.into_bytes())))
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if use_delimiter {
|
||||||
|
let mut token: Option<String> = None;
|
||||||
|
loop {
|
||||||
|
let params = myfsio_common::types::ShallowListParams {
|
||||||
|
prefix: prefix.clone(),
|
||||||
|
delimiter: "/".to_string(),
|
||||||
|
max_keys: UI_OBJECT_BROWSER_MAX_KEYS,
|
||||||
|
continuation_token: token.clone(),
|
||||||
|
};
|
||||||
|
match storage.list_objects_shallow(&bucket, ¶ms).await {
|
||||||
|
Ok(res) => {
|
||||||
|
for p in &res.common_prefixes {
|
||||||
|
let line = json!({ "type": "folder", "prefix": p }).to_string() + "\n";
|
||||||
|
if tx
|
||||||
|
.send(Ok(bytes::Bytes::from(line.into_bytes())))
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for o in &res.objects {
|
||||||
|
let line = json!({
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"key": o.key,
|
"key": o.key,
|
||||||
"size": o.size,
|
"size": o.size,
|
||||||
@@ -1050,38 +1112,46 @@ pub async fn stream_bucket_objects(
|
|||||||
"etag": o.etag.clone().unwrap_or_default(),
|
"etag": o.etag.clone().unwrap_or_default(),
|
||||||
"storage_class": o.storage_class.clone().unwrap_or_else(|| "STANDARD".to_string()),
|
"storage_class": o.storage_class.clone().unwrap_or_else(|| "STANDARD".to_string()),
|
||||||
})
|
})
|
||||||
.to_string(),
|
.to_string()
|
||||||
);
|
+ "\n";
|
||||||
|
if tx
|
||||||
|
.send(Ok(bytes::Bytes::from(line.into_bytes())))
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !res.is_truncated || res.next_continuation_token.is_none() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
token = res.next_continuation_token;
|
||||||
}
|
}
|
||||||
if !res.is_truncated || res.next_continuation_token.is_none() {
|
Err(e) => {
|
||||||
break;
|
let line =
|
||||||
|
json!({ "type": "error", "error": e.to_string() }).to_string() + "\n";
|
||||||
|
let _ = tx.send(Ok(bytes::Bytes::from(line.into_bytes()))).await;
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
token = res.next_continuation_token;
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
lines.push(json!({ "type": "error", "error": e.to_string() }).to_string());
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
} else {
|
||||||
} else {
|
let mut token: Option<String> = None;
|
||||||
let mut token: Option<String> = None;
|
loop {
|
||||||
loop {
|
let params = ListParams {
|
||||||
let params = ListParams {
|
max_keys: 1000,
|
||||||
max_keys: 1000,
|
continuation_token: token.clone(),
|
||||||
continuation_token: token.clone(),
|
prefix: if prefix.is_empty() {
|
||||||
prefix: if prefix.is_empty() {
|
None
|
||||||
None
|
} else {
|
||||||
} else {
|
Some(prefix.clone())
|
||||||
Some(prefix.clone())
|
},
|
||||||
},
|
start_after: None,
|
||||||
start_after: None,
|
};
|
||||||
};
|
match storage.list_objects(&bucket, ¶ms).await {
|
||||||
match state.storage.list_objects(&bucket_name, ¶ms).await {
|
Ok(res) => {
|
||||||
Ok(res) => {
|
for o in &res.objects {
|
||||||
for o in &res.objects {
|
let line = json!({
|
||||||
lines.push(
|
|
||||||
json!({
|
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"key": o.key,
|
"key": o.key,
|
||||||
"size": o.size,
|
"size": o.size,
|
||||||
@@ -1091,33 +1161,154 @@ pub async fn stream_bucket_objects(
|
|||||||
"etag": o.etag.clone().unwrap_or_default(),
|
"etag": o.etag.clone().unwrap_or_default(),
|
||||||
"storage_class": o.storage_class.clone().unwrap_or_else(|| "STANDARD".to_string()),
|
"storage_class": o.storage_class.clone().unwrap_or_else(|| "STANDARD".to_string()),
|
||||||
})
|
})
|
||||||
.to_string(),
|
.to_string()
|
||||||
);
|
+ "\n";
|
||||||
|
if tx
|
||||||
|
.send(Ok(bytes::Bytes::from(line.into_bytes())))
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !res.is_truncated || res.next_continuation_token.is_none() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
token = res.next_continuation_token;
|
||||||
}
|
}
|
||||||
if !res.is_truncated || res.next_continuation_token.is_none() {
|
Err(e) => {
|
||||||
break;
|
let line =
|
||||||
|
json!({ "type": "error", "error": e.to_string() }).to_string() + "\n";
|
||||||
|
let _ = tx.send(Ok(bytes::Bytes::from(line.into_bytes()))).await;
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
token = res.next_continuation_token;
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
lines.push(json!({ "type": "error", "error": e.to_string() }).to_string());
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
lines.push(json!({ "type": "done" }).to_string());
|
let done_line = json!({ "type": "done" }).to_string() + "\n";
|
||||||
|
let _ = tx
|
||||||
|
.send(Ok(bytes::Bytes::from(done_line.into_bytes())))
|
||||||
|
.await;
|
||||||
|
});
|
||||||
|
|
||||||
|
let stream = tokio_stream::wrappers::ReceiverStream::new(rx);
|
||||||
|
let body = Body::from_stream(stream);
|
||||||
|
|
||||||
let body = lines.join("\n") + "\n";
|
|
||||||
let mut headers = HeaderMap::new();
|
let mut headers = HeaderMap::new();
|
||||||
headers.insert(
|
headers.insert(
|
||||||
header::CONTENT_TYPE,
|
header::CONTENT_TYPE,
|
||||||
"application/x-ndjson; charset=utf-8".parse().unwrap(),
|
"application/x-ndjson; charset=utf-8".parse().unwrap(),
|
||||||
);
|
);
|
||||||
|
headers.insert(header::CACHE_CONTROL, "no-cache".parse().unwrap());
|
||||||
|
headers.insert("x-accel-buffering", "no".parse().unwrap());
|
||||||
|
|
||||||
(StatusCode::OK, headers, body).into_response()
|
(StatusCode::OK, headers, body).into_response()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Default)]
|
||||||
|
pub struct SearchObjectsQuery {
|
||||||
|
#[serde(default)]
|
||||||
|
pub q: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub prefix: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub limit: Option<usize>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub start_after: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn search_bucket_objects(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Extension(_session): Extension<SessionHandle>,
|
||||||
|
Path(bucket_name): Path<String>,
|
||||||
|
Query(q): Query<SearchObjectsQuery>,
|
||||||
|
) -> Response {
|
||||||
|
if !matches!(state.storage.bucket_exists(&bucket_name).await, Ok(true)) {
|
||||||
|
return json_error(StatusCode::NOT_FOUND, "Bucket not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
let term = q.q.unwrap_or_default().to_lowercase();
|
||||||
|
let limit = q.limit.unwrap_or(500).clamp(1, 1000);
|
||||||
|
let prefix = q.prefix.clone().unwrap_or_default();
|
||||||
|
let start_after = q.start_after.clone().filter(|s| !s.is_empty());
|
||||||
|
|
||||||
|
if term.is_empty() {
|
||||||
|
return Json(json!({ "results": [], "truncated": false, "next_token": Value::Null }))
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut results: Vec<Value> = Vec::new();
|
||||||
|
let mut truncated = false;
|
||||||
|
let mut last_match_key: Option<String> = None;
|
||||||
|
let mut token: Option<String> = None;
|
||||||
|
let mut start_after_arg = start_after;
|
||||||
|
loop {
|
||||||
|
let params = ListParams {
|
||||||
|
max_keys: 1000,
|
||||||
|
continuation_token: token.clone(),
|
||||||
|
prefix: if prefix.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(prefix.clone())
|
||||||
|
},
|
||||||
|
start_after: start_after_arg.take(),
|
||||||
|
};
|
||||||
|
match state.storage.list_objects(&bucket_name, ¶ms).await {
|
||||||
|
Ok(res) => {
|
||||||
|
for o in &res.objects {
|
||||||
|
if o.key.to_lowercase().contains(&term) {
|
||||||
|
if results.len() >= limit {
|
||||||
|
truncated = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
last_match_key = Some(o.key.clone());
|
||||||
|
results.push(object_json(&bucket_name, o));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if truncated || !res.is_truncated || res.next_continuation_token.is_none() {
|
||||||
|
if res.is_truncated && results.len() >= limit {
|
||||||
|
truncated = true;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
token = res.next_continuation_token;
|
||||||
|
}
|
||||||
|
Err(e) => return storage_json_error(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let next_token = if truncated { last_match_key } else { None };
|
||||||
|
Json(json!({
|
||||||
|
"results": results,
|
||||||
|
"truncated": truncated,
|
||||||
|
"next_token": next_token,
|
||||||
|
}))
|
||||||
|
.into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn bucket_stats_json(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Extension(_session): Extension<SessionHandle>,
|
||||||
|
Path(bucket_name): Path<String>,
|
||||||
|
) -> Response {
|
||||||
|
if !matches!(state.storage.bucket_exists(&bucket_name).await, Ok(true)) {
|
||||||
|
return json_error(StatusCode::NOT_FOUND, "Bucket not found");
|
||||||
|
}
|
||||||
|
match state.storage.bucket_stats(&bucket_name).await {
|
||||||
|
Ok(stats) => Json(json!({
|
||||||
|
"objects": stats.objects,
|
||||||
|
"bytes": stats.bytes,
|
||||||
|
"version_count": stats.version_count,
|
||||||
|
"version_bytes": stats.version_bytes,
|
||||||
|
"total_objects": stats.objects + stats.version_count,
|
||||||
|
"total_bytes": stats.bytes + stats.version_bytes,
|
||||||
|
}))
|
||||||
|
.into_response(),
|
||||||
|
Err(e) => storage_json_error(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn list_bucket_folders(
|
pub async fn list_bucket_folders(
|
||||||
State(state): State<AppState>,
|
State(state): State<AppState>,
|
||||||
Extension(_session): Extension<SessionHandle>,
|
Extension(_session): Extension<SessionHandle>,
|
||||||
@@ -1876,6 +2067,7 @@ pub async fn upload_object(
|
|||||||
State(state),
|
State(state),
|
||||||
Path((bucket_name.clone(), key.clone())),
|
Path((bucket_name.clone(), key.clone())),
|
||||||
Query(ObjectQuery::default()),
|
Query(ObjectQuery::default()),
|
||||||
|
None,
|
||||||
upload_headers,
|
upload_headers,
|
||||||
Body::from(bytes),
|
Body::from(bytes),
|
||||||
)
|
)
|
||||||
@@ -1979,6 +2171,7 @@ pub async fn complete_multipart_upload(
|
|||||||
State(state): State<AppState>,
|
State(state): State<AppState>,
|
||||||
Extension(_session): Extension<SessionHandle>,
|
Extension(_session): Extension<SessionHandle>,
|
||||||
Path((bucket_name, upload_id)): Path<(String, String)>,
|
Path((bucket_name, upload_id)): Path<(String, String)>,
|
||||||
|
headers: HeaderMap,
|
||||||
body: Body,
|
body: Body,
|
||||||
) -> Response {
|
) -> Response {
|
||||||
let payload: CompleteMultipartPayload = match parse_json_body(body).await {
|
let payload: CompleteMultipartPayload = match parse_json_body(body).await {
|
||||||
@@ -1999,6 +2192,22 @@ pub async fn complete_multipart_upload(
|
|||||||
})
|
})
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
let upload_key = match state.storage.list_multipart_uploads(&bucket_name).await {
|
||||||
|
Ok(uploads) => uploads
|
||||||
|
.into_iter()
|
||||||
|
.find(|u| u.upload_id == upload_id)
|
||||||
|
.map(|u| u.key),
|
||||||
|
Err(err) => return storage_json_error(err),
|
||||||
|
};
|
||||||
|
if let Some(ref key) = upload_key {
|
||||||
|
if let Err(response) =
|
||||||
|
super::ensure_archived_null_lock_allows_overwrite(&state, &bucket_name, key, Some(&headers))
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
match state
|
match state
|
||||||
.storage
|
.storage
|
||||||
.complete_multipart(&bucket_name, &upload_id, &parts)
|
.complete_multipart(&bucket_name, &upload_id, &parts)
|
||||||
@@ -2236,7 +2445,11 @@ async fn object_metadata_json(state: &AppState, bucket: &str, key: &str) -> Resp
|
|||||||
.await
|
.await
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
let mut out = metadata.clone();
|
let mut out: std::collections::HashMap<String, String> = metadata
|
||||||
|
.iter()
|
||||||
|
.filter(|(k, _)| !(k.starts_with("__") && k.ends_with("__")))
|
||||||
|
.map(|(k, v)| (k.clone(), v.clone()))
|
||||||
|
.collect();
|
||||||
if let Some(content_type) = head.content_type {
|
if let Some(content_type) = head.content_type {
|
||||||
out.insert("Content-Type".to_string(), content_type);
|
out.insert("Content-Type".to_string(), content_type);
|
||||||
}
|
}
|
||||||
@@ -2459,7 +2672,13 @@ struct CopyMovePayload {
|
|||||||
dest_key: String,
|
dest_key: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn copy_object_json(state: &AppState, bucket: &str, key: &str, body: Body) -> Response {
|
async fn copy_object_json(
|
||||||
|
state: &AppState,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
headers: &HeaderMap,
|
||||||
|
body: Body,
|
||||||
|
) -> Response {
|
||||||
let payload: CopyMovePayload = match parse_json_body(body).await {
|
let payload: CopyMovePayload = match parse_json_body(body).await {
|
||||||
Ok(payload) => payload,
|
Ok(payload) => payload,
|
||||||
Err(response) => return response,
|
Err(response) => return response,
|
||||||
@@ -2473,6 +2692,17 @@ async fn copy_object_json(state: &AppState, bucket: &str, key: &str, body: Body)
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if let Err(response) = super::ensure_archived_null_lock_allows_overwrite(
|
||||||
|
state,
|
||||||
|
dest_bucket,
|
||||||
|
dest_key,
|
||||||
|
Some(headers),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
match state
|
match state
|
||||||
.storage
|
.storage
|
||||||
.copy_object(bucket, key, dest_bucket, dest_key)
|
.copy_object(bucket, key, dest_bucket, dest_key)
|
||||||
@@ -2492,7 +2722,13 @@ async fn copy_object_json(state: &AppState, bucket: &str, key: &str, body: Body)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn move_object_json(state: &AppState, bucket: &str, key: &str, body: Body) -> Response {
|
async fn move_object_json(
|
||||||
|
state: &AppState,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
headers: &HeaderMap,
|
||||||
|
body: Body,
|
||||||
|
) -> Response {
|
||||||
let payload: CopyMovePayload = match parse_json_body(body).await {
|
let payload: CopyMovePayload = match parse_json_body(body).await {
|
||||||
Ok(payload) => payload,
|
Ok(payload) => payload,
|
||||||
Err(response) => return response,
|
Err(response) => return response,
|
||||||
@@ -2512,9 +2748,20 @@ async fn move_object_json(state: &AppState, bucket: &str, key: &str, body: Body)
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if let Err(response) = super::ensure_archived_null_lock_allows_overwrite(
|
||||||
|
state,
|
||||||
|
dest_bucket,
|
||||||
|
dest_key,
|
||||||
|
Some(headers),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
match state.storage.copy_object(bucket, key, dest_bucket, dest_key).await {
|
match state.storage.copy_object(bucket, key, dest_bucket, dest_key).await {
|
||||||
Ok(_) => match state.storage.delete_object(bucket, key).await {
|
Ok(_) => match state.storage.delete_object(bucket, key).await {
|
||||||
Ok(()) => {
|
Ok(_) => {
|
||||||
super::trigger_replication(state, dest_bucket, dest_key, "write");
|
super::trigger_replication(state, dest_bucket, dest_key, "write");
|
||||||
super::trigger_replication(state, bucket, key, "delete");
|
super::trigger_replication(state, bucket, key, "delete");
|
||||||
Json(json!({
|
Json(json!({
|
||||||
@@ -2589,7 +2836,7 @@ async fn delete_object_json(
|
|||||||
}
|
}
|
||||||
|
|
||||||
match state.storage.delete_object(bucket, key).await {
|
match state.storage.delete_object(bucket, key).await {
|
||||||
Ok(()) => {
|
Ok(_) => {
|
||||||
super::trigger_replication(state, bucket, key, "delete");
|
super::trigger_replication(state, bucket, key, "delete");
|
||||||
Json(json!({
|
Json(json!({
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
@@ -2774,8 +3021,12 @@ pub async fn object_post_dispatch(
|
|||||||
object_presign_json(&state, &session, &bucket_name, &key, body).await
|
object_presign_json(&state, &session, &bucket_name, &key, body).await
|
||||||
}
|
}
|
||||||
ObjectPostAction::Tags => update_object_tags(&state, &bucket_name, &key, body).await,
|
ObjectPostAction::Tags => update_object_tags(&state, &bucket_name, &key, body).await,
|
||||||
ObjectPostAction::Copy => copy_object_json(&state, &bucket_name, &key, body).await,
|
ObjectPostAction::Copy => {
|
||||||
ObjectPostAction::Move => move_object_json(&state, &bucket_name, &key, body).await,
|
copy_object_json(&state, &bucket_name, &key, &headers, body).await
|
||||||
|
}
|
||||||
|
ObjectPostAction::Move => {
|
||||||
|
move_object_json(&state, &bucket_name, &key, &headers, body).await
|
||||||
|
}
|
||||||
ObjectPostAction::Restore(version_id) => {
|
ObjectPostAction::Restore(version_id) => {
|
||||||
restore_object_version_json(&state, &bucket_name, &key, &version_id).await
|
restore_object_version_json(&state, &bucket_name, &key, &version_id).await
|
||||||
}
|
}
|
||||||
@@ -2868,7 +3119,7 @@ pub async fn bulk_delete_objects(
|
|||||||
|
|
||||||
for key in keys {
|
for key in keys {
|
||||||
match state.storage.delete_object(&bucket_name, &key).await {
|
match state.storage.delete_object(&bucket_name, &key).await {
|
||||||
Ok(()) => {
|
Ok(_) => {
|
||||||
super::trigger_replication(&state, &bucket_name, &key, "delete");
|
super::trigger_replication(&state, &bucket_name, &key, "delete");
|
||||||
if payload.purge_versions {
|
if payload.purge_versions {
|
||||||
if let Err(err) =
|
if let Err(err) =
|
||||||
|
|||||||
@@ -123,6 +123,7 @@ pub fn register_ui_endpoints(engine: &TemplateEngine) {
|
|||||||
("ui.sites_dashboard", "/ui/sites"),
|
("ui.sites_dashboard", "/ui/sites"),
|
||||||
("ui.update_local_site", "/ui/sites/local"),
|
("ui.update_local_site", "/ui/sites/local"),
|
||||||
("ui.add_peer_site", "/ui/sites/peers"),
|
("ui.add_peer_site", "/ui/sites/peers"),
|
||||||
|
("ui.cluster_dashboard", "/ui/cluster"),
|
||||||
("ui.metrics_dashboard", "/ui/metrics"),
|
("ui.metrics_dashboard", "/ui/metrics"),
|
||||||
("ui.system_dashboard", "/ui/system"),
|
("ui.system_dashboard", "/ui/system"),
|
||||||
("ui.system_gc_status", "/ui/system/gc/status"),
|
("ui.system_gc_status", "/ui/system/gc/status"),
|
||||||
@@ -227,9 +228,7 @@ async fn parse_form_any(
|
|||||||
if is_multipart {
|
if is_multipart {
|
||||||
let boundary = multer::parse_boundary(&content_type)
|
let boundary = multer::parse_boundary(&content_type)
|
||||||
.map_err(|_| "Missing multipart boundary".to_string())?;
|
.map_err(|_| "Missing multipart boundary".to_string())?;
|
||||||
let stream = futures::stream::once(async move {
|
let stream = futures::stream::once(async move { Ok::<_, std::io::Error>(bytes) });
|
||||||
Ok::<_, std::io::Error>(bytes)
|
|
||||||
});
|
|
||||||
let mut multipart = multer::Multipart::new(stream, boundary);
|
let mut multipart = multer::Multipart::new(stream, boundary);
|
||||||
let mut out = HashMap::new();
|
let mut out = HashMap::new();
|
||||||
while let Some(field) = multipart
|
while let Some(field) = multipart
|
||||||
@@ -398,7 +397,13 @@ pub async fn bucket_detail(
|
|||||||
Query(request_args): Query<HashMap<String, String>>,
|
Query(request_args): Query<HashMap<String, String>>,
|
||||||
) -> Response {
|
) -> Response {
|
||||||
if !matches!(state.storage.bucket_exists(&bucket_name).await, Ok(true)) {
|
if !matches!(state.storage.bucket_exists(&bucket_name).await, Ok(true)) {
|
||||||
return (StatusCode::NOT_FOUND, "Bucket not found").into_response();
|
session.write(|s| {
|
||||||
|
s.push_flash(
|
||||||
|
"danger",
|
||||||
|
format!("Bucket '{}' does not exist.", bucket_name),
|
||||||
|
)
|
||||||
|
});
|
||||||
|
return Redirect::to("/ui/buckets").into_response();
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut ctx = page_context(&state, &session, "ui.bucket_detail");
|
let mut ctx = page_context(&state, &session, "ui.bucket_detail");
|
||||||
@@ -423,11 +428,19 @@ pub async fn bucket_detail(
|
|||||||
let target_conn = replication_rule
|
let target_conn = replication_rule
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|rule| state.connections.get(&rule.target_connection_id));
|
.and_then(|rule| state.connections.get(&rule.target_connection_id));
|
||||||
let versioning_enabled = state
|
let versioning_status_enum = state
|
||||||
.storage
|
.storage
|
||||||
.is_versioning_enabled(&bucket_name)
|
.get_versioning_status(&bucket_name)
|
||||||
.await
|
.await
|
||||||
.unwrap_or(false);
|
.unwrap_or(myfsio_common::types::VersioningStatus::Disabled);
|
||||||
|
let versioning_enabled = matches!(
|
||||||
|
versioning_status_enum,
|
||||||
|
myfsio_common::types::VersioningStatus::Enabled
|
||||||
|
);
|
||||||
|
let versioning_suspended = matches!(
|
||||||
|
versioning_status_enum,
|
||||||
|
myfsio_common::types::VersioningStatus::Suspended
|
||||||
|
);
|
||||||
let encryption_config = config_encryption_to_ui(bucket_config.encryption.as_ref());
|
let encryption_config = config_encryption_to_ui(bucket_config.encryption.as_ref());
|
||||||
let website_config = config_website_to_ui(bucket_config.website.as_ref());
|
let website_config = config_website_to_ui(bucket_config.website.as_ref());
|
||||||
let quota = bucket_config.quota.clone();
|
let quota = bucket_config.quota.clone();
|
||||||
@@ -487,12 +500,13 @@ pub async fn bucket_detail(
|
|||||||
);
|
);
|
||||||
ctx.insert("has_quota", "a.is_some());
|
ctx.insert("has_quota", "a.is_some());
|
||||||
ctx.insert("versioning_enabled", &versioning_enabled);
|
ctx.insert("versioning_enabled", &versioning_enabled);
|
||||||
|
ctx.insert("versioning_suspended", &versioning_suspended);
|
||||||
ctx.insert(
|
ctx.insert(
|
||||||
"versioning_status",
|
"versioning_status",
|
||||||
&(if versioning_enabled {
|
&(match versioning_status_enum {
|
||||||
"Enabled"
|
myfsio_common::types::VersioningStatus::Enabled => "Enabled",
|
||||||
} else {
|
myfsio_common::types::VersioningStatus::Suspended => "Suspended",
|
||||||
"Disabled"
|
myfsio_common::types::VersioningStatus::Disabled => "Disabled",
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
ctx.insert("encryption_config", &encryption_config);
|
ctx.insert("encryption_config", &encryption_config);
|
||||||
@@ -707,13 +721,21 @@ pub async fn iam_dashboard(
|
|||||||
.as_array()
|
.as_array()
|
||||||
.map(|items| {
|
.map(|items| {
|
||||||
items.iter().any(|policy| {
|
items.iter().any(|policy| {
|
||||||
|
let bucket_wildcard = policy
|
||||||
|
.get("bucket")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|b| b == "*")
|
||||||
|
.unwrap_or(false);
|
||||||
|
if !bucket_wildcard {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
policy
|
policy
|
||||||
.get("actions")
|
.get("actions")
|
||||||
.and_then(|value| value.as_array())
|
.and_then(|value| value.as_array())
|
||||||
.map(|actions| {
|
.map(|actions| {
|
||||||
actions
|
actions
|
||||||
.iter()
|
.iter()
|
||||||
.any(|action| matches!(action.as_str(), Some("*") | Some("iam:*")))
|
.any(|action| action.as_str() == Some("*"))
|
||||||
})
|
})
|
||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
})
|
})
|
||||||
@@ -1178,6 +1200,7 @@ pub async fn sites_dashboard(
|
|||||||
"region": p.region,
|
"region": p.region,
|
||||||
"priority": p.priority,
|
"priority": p.priority,
|
||||||
"connection_id": p.connection_id,
|
"connection_id": p.connection_id,
|
||||||
|
"peer_inbound_access_key": p.peer_inbound_access_key,
|
||||||
"is_healthy": p.is_healthy,
|
"is_healthy": p.is_healthy,
|
||||||
"last_health_check": p.last_health_check,
|
"last_health_check": p.last_health_check,
|
||||||
})
|
})
|
||||||
@@ -1186,20 +1209,60 @@ pub async fn sites_dashboard(
|
|||||||
})
|
})
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let rules = state.replication.rules_snapshot();
|
||||||
|
let sync_snapshot = state
|
||||||
|
.site_sync
|
||||||
|
.as_ref()
|
||||||
|
.map(|w| w.snapshot_stats())
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
let peers_with_stats: Vec<Value> = peers
|
let peers_with_stats: Vec<Value> = peers
|
||||||
.iter()
|
.iter()
|
||||||
.cloned()
|
.cloned()
|
||||||
.map(|peer| {
|
.map(|peer| {
|
||||||
let has_connection = peer
|
let connection_id = peer
|
||||||
.get("connection_id")
|
.get("connection_id")
|
||||||
.and_then(|value| value.as_str())
|
.and_then(|value| value.as_str())
|
||||||
.map(|value| !value.is_empty())
|
.filter(|value| !value.is_empty())
|
||||||
.unwrap_or(false);
|
.map(|value| value.to_string());
|
||||||
|
let has_connection = connection_id.is_some();
|
||||||
|
|
||||||
|
let mut buckets_syncing: u64 = 0;
|
||||||
|
let mut has_bidirectional = false;
|
||||||
|
let mut last_sync_at: Option<f64> = None;
|
||||||
|
let mut total_pulled: u64 = 0;
|
||||||
|
let mut total_errors: u64 = 0;
|
||||||
|
|
||||||
|
if let Some(ref conn_id) = connection_id {
|
||||||
|
for (bucket, rule) in &rules {
|
||||||
|
if &rule.target_connection_id != conn_id || !rule.enabled {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if rule.mode == crate::services::replication::MODE_BIDIRECTIONAL {
|
||||||
|
has_bidirectional = true;
|
||||||
|
buckets_syncing += 1;
|
||||||
|
if let Some(stats) = sync_snapshot.get(bucket) {
|
||||||
|
total_pulled += stats.objects_pulled;
|
||||||
|
total_errors += stats.errors;
|
||||||
|
if let Some(ts) = stats.last_sync_at {
|
||||||
|
last_sync_at = match last_sync_at {
|
||||||
|
Some(prev) if prev > ts => Some(prev),
|
||||||
|
_ => Some(ts),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
json!({
|
json!({
|
||||||
"peer": peer,
|
"peer": peer,
|
||||||
"has_connection": has_connection,
|
"has_connection": has_connection,
|
||||||
"buckets_syncing": 0,
|
"buckets_syncing": buckets_syncing,
|
||||||
"has_bidirectional": false,
|
"has_bidirectional": has_bidirectional,
|
||||||
|
"last_sync_at": last_sync_at,
|
||||||
|
"objects_pulled": total_pulled,
|
||||||
|
"errors": total_errors,
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
@@ -1236,6 +1299,184 @@ pub async fn sites_dashboard(
|
|||||||
render(&state, "sites.html", &ctx)
|
render(&state, "sites.html", &ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn cluster_data_json(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Extension(_session): Extension<SessionHandle>,
|
||||||
|
Query(params): Query<HashMap<String, String>>,
|
||||||
|
) -> Response {
|
||||||
|
let force = params
|
||||||
|
.get("force")
|
||||||
|
.map(|v| v == "1" || v.eq_ignore_ascii_case("true"))
|
||||||
|
.unwrap_or(false);
|
||||||
|
if force {
|
||||||
|
*state.cluster_aggregate_cache.lock() = None;
|
||||||
|
*state.cluster_overview_cache.lock() = None;
|
||||||
|
}
|
||||||
|
let sites = build_cluster_sites(&state).await;
|
||||||
|
let totals = cluster_totals(&sites);
|
||||||
|
let body = json!({ "sites": sites, "totals": totals });
|
||||||
|
(
|
||||||
|
StatusCode::OK,
|
||||||
|
[(header::CONTENT_TYPE, "application/json")],
|
||||||
|
body.to_string(),
|
||||||
|
)
|
||||||
|
.into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cluster_totals(sites: &[Value]) -> Value {
|
||||||
|
let total_buckets: u64 = sites
|
||||||
|
.iter()
|
||||||
|
.filter_map(|s| s.get("buckets").and_then(|v| v.as_u64()))
|
||||||
|
.sum();
|
||||||
|
let total_objects: u64 = sites
|
||||||
|
.iter()
|
||||||
|
.filter_map(|s| s.get("objects").and_then(|v| v.as_u64()))
|
||||||
|
.sum();
|
||||||
|
let total_size_bytes: u64 = sites
|
||||||
|
.iter()
|
||||||
|
.filter_map(|s| s.get("size_bytes").and_then(|v| v.as_u64()))
|
||||||
|
.sum();
|
||||||
|
let online = sites
|
||||||
|
.iter()
|
||||||
|
.filter(|s| s.get("online").and_then(|v| v.as_bool()).unwrap_or(false))
|
||||||
|
.count();
|
||||||
|
json!({
|
||||||
|
"buckets": total_buckets,
|
||||||
|
"objects": total_objects,
|
||||||
|
"size_bytes": total_size_bytes,
|
||||||
|
"online_count": online,
|
||||||
|
"total_count": sites.len(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn cluster_dashboard(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Extension(session): Extension<SessionHandle>,
|
||||||
|
) -> Response {
|
||||||
|
let mut ctx = page_context(&state, &session, "ui.cluster_dashboard");
|
||||||
|
|
||||||
|
let sites = build_cluster_sites(&state).await;
|
||||||
|
|
||||||
|
let total_buckets: u64 = sites
|
||||||
|
.iter()
|
||||||
|
.filter_map(|s| s.get("buckets").and_then(|v| v.as_u64()))
|
||||||
|
.sum();
|
||||||
|
let total_objects: u64 = sites
|
||||||
|
.iter()
|
||||||
|
.filter_map(|s| s.get("objects").and_then(|v| v.as_u64()))
|
||||||
|
.sum();
|
||||||
|
let total_size_bytes: u64 = sites
|
||||||
|
.iter()
|
||||||
|
.filter_map(|s| s.get("size_bytes").and_then(|v| v.as_u64()))
|
||||||
|
.sum();
|
||||||
|
let online_count = sites
|
||||||
|
.iter()
|
||||||
|
.filter(|s| s.get("online").and_then(|v| v.as_bool()).unwrap_or(false))
|
||||||
|
.count();
|
||||||
|
|
||||||
|
ctx.insert("cluster_sites", &sites);
|
||||||
|
ctx.insert("cluster_total_buckets", &total_buckets);
|
||||||
|
ctx.insert("cluster_total_objects", &total_objects);
|
||||||
|
ctx.insert("cluster_total_size_bytes", &total_size_bytes);
|
||||||
|
ctx.insert("cluster_online_count", &online_count);
|
||||||
|
ctx.insert("cluster_total_count", &sites.len());
|
||||||
|
render(&state, "cluster.html", &ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn build_cluster_sites(state: &AppState) -> Vec<Value> {
|
||||||
|
{
|
||||||
|
let guard = state.cluster_aggregate_cache.lock();
|
||||||
|
if let Some((at, ref value)) = *guard {
|
||||||
|
if at.elapsed() < std::time::Duration::from_secs(10) {
|
||||||
|
if let Some(arr) = value.as_array() {
|
||||||
|
return arr.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut sites: Vec<Value> = Vec::new();
|
||||||
|
|
||||||
|
let local = crate::handlers::admin::build_cluster_overview_public(state).await;
|
||||||
|
let mut local_card = decorate_site(local, true, false, None);
|
||||||
|
if local_card.get("site_id").and_then(|v| v.as_str()).is_none() {
|
||||||
|
local_card["site_id"] = json!(state
|
||||||
|
.config
|
||||||
|
.site_id
|
||||||
|
.clone()
|
||||||
|
.unwrap_or_else(|| "local".to_string()));
|
||||||
|
}
|
||||||
|
local_card["is_local"] = json!(true);
|
||||||
|
sites.push(local_card);
|
||||||
|
|
||||||
|
let peers = state
|
||||||
|
.site_registry
|
||||||
|
.as_ref()
|
||||||
|
.map(|r| r.list_peers())
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let connect_to = std::time::Duration::from_secs(2);
|
||||||
|
let read_to = std::time::Duration::from_secs(3);
|
||||||
|
let client = crate::services::peer_admin::PeerAdminClient::new(connect_to, read_to);
|
||||||
|
|
||||||
|
let mut peer_futures = Vec::new();
|
||||||
|
for peer in peers {
|
||||||
|
let conn = peer
|
||||||
|
.connection_id
|
||||||
|
.as_deref()
|
||||||
|
.and_then(|id| state.connections.get(id));
|
||||||
|
let endpoint = peer.endpoint.clone();
|
||||||
|
let conn_clone = conn.clone();
|
||||||
|
let client_ref = &client;
|
||||||
|
peer_futures.push(async move {
|
||||||
|
let value = match conn_clone {
|
||||||
|
Some(c) => client_ref.fetch_cluster_overview(&endpoint, &c).await,
|
||||||
|
None => Err("no connection configured".to_string()),
|
||||||
|
};
|
||||||
|
(peer, value)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let results = futures::future::join_all(peer_futures).await;
|
||||||
|
for (peer, result) in results {
|
||||||
|
let (overview, online, error) = match result {
|
||||||
|
Ok(value) => (value, true, None),
|
||||||
|
Err(err) => (json!({}), false, Some(err)),
|
||||||
|
};
|
||||||
|
let mut card = decorate_site(overview, online, !online, error);
|
||||||
|
if card.get("site_id").and_then(|v| v.as_str()).is_none() {
|
||||||
|
card["site_id"] = json!(peer.site_id.clone());
|
||||||
|
}
|
||||||
|
if card.get("display_name").and_then(|v| v.as_str()).is_none() {
|
||||||
|
card["display_name"] = json!(peer.display_name.clone());
|
||||||
|
}
|
||||||
|
if card.get("endpoint").and_then(|v| v.as_str()).is_none() {
|
||||||
|
card["endpoint"] = json!(peer.endpoint.clone());
|
||||||
|
}
|
||||||
|
card["is_local"] = json!(false);
|
||||||
|
card["registered_priority"] = json!(peer.priority);
|
||||||
|
card["registered_region"] = json!(peer.region);
|
||||||
|
sites.push(card);
|
||||||
|
}
|
||||||
|
|
||||||
|
*state.cluster_aggregate_cache.lock() =
|
||||||
|
Some((std::time::Instant::now(), Value::Array(sites.clone())));
|
||||||
|
sites
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decorate_site(mut value: Value, online: bool, stale: bool, error: Option<String>) -> Value {
|
||||||
|
if !value.is_object() {
|
||||||
|
value = json!({});
|
||||||
|
}
|
||||||
|
value["online"] = json!(online);
|
||||||
|
value["stale"] = json!(stale);
|
||||||
|
value["error"] = match error {
|
||||||
|
Some(e) => json!(e),
|
||||||
|
None => Value::Null,
|
||||||
|
};
|
||||||
|
value
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(serde::Deserialize)]
|
#[derive(serde::Deserialize)]
|
||||||
pub struct LocalSiteForm {
|
pub struct LocalSiteForm {
|
||||||
pub site_id: String,
|
pub site_id: String,
|
||||||
@@ -1264,6 +1505,8 @@ pub struct PeerSiteForm {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub connection_id: String,
|
pub connection_id: String,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
|
pub peer_inbound_access_key: String,
|
||||||
|
#[serde(default)]
|
||||||
pub csrf_token: String,
|
pub csrf_token: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1425,6 +1668,14 @@ pub async fn add_peer_site(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let has_connection = connection_id.is_some();
|
let has_connection = connection_id.is_some();
|
||||||
|
let peer_inbound_access_key = {
|
||||||
|
let value = form.peer_inbound_access_key.trim();
|
||||||
|
if value.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(value.to_string())
|
||||||
|
}
|
||||||
|
};
|
||||||
let peer = crate::services::site_registry::PeerSite {
|
let peer = crate::services::site_registry::PeerSite {
|
||||||
site_id: site_id.clone(),
|
site_id: site_id.clone(),
|
||||||
endpoint,
|
endpoint,
|
||||||
@@ -1439,6 +1690,7 @@ pub async fn add_peer_site(
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
connection_id: connection_id.clone(),
|
connection_id: connection_id.clone(),
|
||||||
|
peer_inbound_access_key,
|
||||||
created_at: None,
|
created_at: None,
|
||||||
is_healthy: false,
|
is_healthy: false,
|
||||||
last_health_check: None,
|
last_health_check: None,
|
||||||
@@ -1500,6 +1752,20 @@ pub async fn update_peer_site(
|
|||||||
return Redirect::to("/ui/sites").into_response();
|
return Redirect::to("/ui/sites").into_response();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let endpoint = form.endpoint.trim().to_string();
|
||||||
|
if endpoint.is_empty() {
|
||||||
|
let message = "Endpoint is required.".to_string();
|
||||||
|
if wants_json {
|
||||||
|
return (
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
axum::Json(json!({ "error": message })),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
session.write(|s| s.push_flash("danger", message));
|
||||||
|
return Redirect::to("/ui/sites").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
let connection_id = {
|
let connection_id = {
|
||||||
let value = form.connection_id.trim();
|
let value = form.connection_id.trim();
|
||||||
if value.is_empty() {
|
if value.is_empty() {
|
||||||
@@ -1523,9 +1789,17 @@ pub async fn update_peer_site(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let peer_inbound_access_key = {
|
||||||
|
let value = form.peer_inbound_access_key.trim();
|
||||||
|
if value.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(value.to_string())
|
||||||
|
}
|
||||||
|
};
|
||||||
let peer = crate::services::site_registry::PeerSite {
|
let peer = crate::services::site_registry::PeerSite {
|
||||||
site_id: site_id.clone(),
|
site_id: site_id.clone(),
|
||||||
endpoint: form.endpoint.trim().to_string(),
|
endpoint,
|
||||||
region: form.region.trim().to_string(),
|
region: form.region.trim().to_string(),
|
||||||
priority: form.priority,
|
priority: form.priority,
|
||||||
display_name: {
|
display_name: {
|
||||||
@@ -1537,6 +1811,7 @@ pub async fn update_peer_site(
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
connection_id,
|
connection_id,
|
||||||
|
peer_inbound_access_key,
|
||||||
created_at: existing.created_at,
|
created_at: existing.created_at,
|
||||||
is_healthy: existing.is_healthy,
|
is_healthy: existing.is_healthy,
|
||||||
last_health_check: existing.last_health_check,
|
last_health_check: existing.last_health_check,
|
||||||
@@ -1676,13 +1951,17 @@ pub async fn metrics_dashboard(
|
|||||||
render(&state, "metrics.html", &ctx)
|
render(&state, "metrics.html", &ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn format_history_timestamp(timestamp: Option<f64>) -> String {
|
fn format_history_timestamp(timestamp: Option<f64>, tz: chrono_tz::Tz) -> String {
|
||||||
let Some(timestamp) = timestamp else {
|
let Some(timestamp) = timestamp else {
|
||||||
return "-".to_string();
|
return "-".to_string();
|
||||||
};
|
};
|
||||||
let millis = (timestamp * 1000.0).round() as i64;
|
let millis = (timestamp * 1000.0).round() as i64;
|
||||||
chrono::DateTime::<chrono::Utc>::from_timestamp_millis(millis)
|
chrono::DateTime::<chrono::Utc>::from_timestamp_millis(millis)
|
||||||
.map(|dt| dt.format("%Y-%m-%d %H:%M:%S UTC").to_string())
|
.map(|dt| {
|
||||||
|
dt.with_timezone(&tz)
|
||||||
|
.format("%Y-%m-%d %H:%M:%S %Z")
|
||||||
|
.to_string()
|
||||||
|
})
|
||||||
.unwrap_or_else(|| "-".to_string())
|
.unwrap_or_else(|| "-".to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1701,7 +1980,7 @@ fn format_byte_count(bytes: u64) -> String {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn decorate_gc_history(executions: &[Value]) -> Vec<Value> {
|
fn decorate_gc_history(executions: &[Value], tz: chrono_tz::Tz) -> Vec<Value> {
|
||||||
executions
|
executions
|
||||||
.iter()
|
.iter()
|
||||||
.cloned()
|
.cloned()
|
||||||
@@ -1715,7 +1994,7 @@ fn decorate_gc_history(executions: &[Value]) -> Vec<Value> {
|
|||||||
if let Some(obj) = execution.as_object_mut() {
|
if let Some(obj) = execution.as_object_mut() {
|
||||||
obj.insert(
|
obj.insert(
|
||||||
"timestamp_display".to_string(),
|
"timestamp_display".to_string(),
|
||||||
Value::String(format_history_timestamp(timestamp)),
|
Value::String(format_history_timestamp(timestamp, tz)),
|
||||||
);
|
);
|
||||||
obj.insert(
|
obj.insert(
|
||||||
"bytes_freed_display".to_string(),
|
"bytes_freed_display".to_string(),
|
||||||
@@ -1727,7 +2006,7 @@ fn decorate_gc_history(executions: &[Value]) -> Vec<Value> {
|
|||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn decorate_integrity_history(executions: &[Value]) -> Vec<Value> {
|
fn decorate_integrity_history(executions: &[Value], tz: chrono_tz::Tz) -> Vec<Value> {
|
||||||
executions
|
executions
|
||||||
.iter()
|
.iter()
|
||||||
.cloned()
|
.cloned()
|
||||||
@@ -1736,7 +2015,7 @@ fn decorate_integrity_history(executions: &[Value]) -> Vec<Value> {
|
|||||||
if let Some(obj) = execution.as_object_mut() {
|
if let Some(obj) = execution.as_object_mut() {
|
||||||
obj.insert(
|
obj.insert(
|
||||||
"timestamp_display".to_string(),
|
"timestamp_display".to_string(),
|
||||||
Value::String(format_history_timestamp(timestamp)),
|
Value::String(format_history_timestamp(timestamp, tz)),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
execution
|
execution
|
||||||
@@ -1749,6 +2028,11 @@ pub async fn system_dashboard(
|
|||||||
Extension(session): Extension<SessionHandle>,
|
Extension(session): Extension<SessionHandle>,
|
||||||
) -> Response {
|
) -> Response {
|
||||||
let mut ctx = page_context(&state, &session, "ui.system_dashboard");
|
let mut ctx = page_context(&state, &session, "ui.system_dashboard");
|
||||||
|
let display_tz: chrono_tz::Tz = state
|
||||||
|
.config
|
||||||
|
.display_timezone
|
||||||
|
.parse()
|
||||||
|
.unwrap_or(chrono_tz::UTC);
|
||||||
|
|
||||||
let gc_status = match &state.gc {
|
let gc_status = match &state.gc {
|
||||||
Some(gc) => gc.status().await,
|
Some(gc) => gc.status().await,
|
||||||
@@ -1770,7 +2054,7 @@ pub async fn system_dashboard(
|
|||||||
.await
|
.await
|
||||||
.get("executions")
|
.get("executions")
|
||||||
.and_then(|value| value.as_array())
|
.and_then(|value| value.as_array())
|
||||||
.map(|values| decorate_gc_history(values))
|
.map(|values| decorate_gc_history(values, display_tz))
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
None => Vec::new(),
|
None => Vec::new(),
|
||||||
};
|
};
|
||||||
@@ -1794,7 +2078,7 @@ pub async fn system_dashboard(
|
|||||||
.await
|
.await
|
||||||
.get("executions")
|
.get("executions")
|
||||||
.and_then(|value| value.as_array())
|
.and_then(|value| value.as_array())
|
||||||
.map(|values| decorate_integrity_history(values))
|
.map(|values| decorate_integrity_history(values, display_tz))
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
None => Vec::new(),
|
None => Vec::new(),
|
||||||
};
|
};
|
||||||
@@ -1806,7 +2090,7 @@ pub async fn system_dashboard(
|
|||||||
ctx.insert("gc_status", &gc_status);
|
ctx.insert("gc_status", &gc_status);
|
||||||
ctx.insert("integrity_status", &integrity_status);
|
ctx.insert("integrity_status", &integrity_status);
|
||||||
ctx.insert("app_version", &env!("CARGO_PKG_VERSION"));
|
ctx.insert("app_version", &env!("CARGO_PKG_VERSION"));
|
||||||
ctx.insert("display_timezone", &"UTC");
|
ctx.insert("display_timezone", &state.config.display_timezone);
|
||||||
ctx.insert("platform", &std::env::consts::OS);
|
ctx.insert("platform", &std::env::consts::OS);
|
||||||
ctx.insert(
|
ctx.insert(
|
||||||
"storage_root",
|
"storage_root",
|
||||||
@@ -2173,10 +2457,7 @@ pub async fn create_bucket(
|
|||||||
let wants_json = wants_json(&headers);
|
let wants_json = wants_json(&headers);
|
||||||
let form = match parse_form_any(&headers, body).await {
|
let form = match parse_form_any(&headers, body).await {
|
||||||
Ok(fields) => CreateBucketForm {
|
Ok(fields) => CreateBucketForm {
|
||||||
bucket_name: fields
|
bucket_name: fields.get("bucket_name").cloned().unwrap_or_default(),
|
||||||
.get("bucket_name")
|
|
||||||
.cloned()
|
|
||||||
.unwrap_or_default(),
|
|
||||||
csrf_token: fields.get("csrf_token").cloned().unwrap_or_default(),
|
csrf_token: fields.get("csrf_token").cloned().unwrap_or_default(),
|
||||||
},
|
},
|
||||||
Err(message) => {
|
Err(message) => {
|
||||||
@@ -2360,10 +2641,10 @@ pub async fn update_bucket_replication(
|
|||||||
"pause" => {
|
"pause" => {
|
||||||
let Some(mut rule) = state.replication.get_rule(&bucket_name) else {
|
let Some(mut rule) = state.replication.get_rule(&bucket_name) else {
|
||||||
return respond(
|
return respond(
|
||||||
false,
|
true,
|
||||||
StatusCode::NOT_FOUND,
|
StatusCode::OK,
|
||||||
"No replication configuration to pause.".to_string(),
|
"No replication configuration to pause.".to_string(),
|
||||||
json!({ "error": "No replication configuration to pause" }),
|
json!({ "action": "pause", "enabled": false, "no_op": true }),
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
rule.enabled = false;
|
rule.enabled = false;
|
||||||
@@ -2378,10 +2659,10 @@ pub async fn update_bucket_replication(
|
|||||||
"resume" => {
|
"resume" => {
|
||||||
let Some(mut rule) = state.replication.get_rule(&bucket_name) else {
|
let Some(mut rule) = state.replication.get_rule(&bucket_name) else {
|
||||||
return respond(
|
return respond(
|
||||||
false,
|
true,
|
||||||
StatusCode::NOT_FOUND,
|
StatusCode::OK,
|
||||||
"No replication configuration to resume.".to_string(),
|
"No replication configuration to resume.".to_string(),
|
||||||
json!({ "error": "No replication configuration to resume" }),
|
json!({ "action": "resume", "enabled": false, "no_op": true }),
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
rule.enabled = true;
|
rule.enabled = true;
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
pub mod config;
|
pub mod config;
|
||||||
|
pub mod embedded;
|
||||||
pub mod handlers;
|
pub mod handlers;
|
||||||
pub mod middleware;
|
pub mod middleware;
|
||||||
pub mod services;
|
pub mod services;
|
||||||
@@ -91,6 +92,14 @@ pub fn create_ui_router(state: state::AppState) -> Router {
|
|||||||
"/ui/buckets/{bucket_name}/objects/stream",
|
"/ui/buckets/{bucket_name}/objects/stream",
|
||||||
get(ui_api::stream_bucket_objects),
|
get(ui_api::stream_bucket_objects),
|
||||||
)
|
)
|
||||||
|
.route(
|
||||||
|
"/ui/buckets/{bucket_name}/objects/search",
|
||||||
|
get(ui_api::search_bucket_objects),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/ui/buckets/{bucket_name}/stats",
|
||||||
|
get(ui_api::bucket_stats_json),
|
||||||
|
)
|
||||||
.route(
|
.route(
|
||||||
"/ui/buckets/{bucket_name}/folders",
|
"/ui/buckets/{bucket_name}/folders",
|
||||||
get(ui_api::list_bucket_folders),
|
get(ui_api::list_bucket_folders),
|
||||||
@@ -222,6 +231,8 @@ pub fn create_ui_router(state: state::AppState) -> Router {
|
|||||||
get(ui_api::connection_health),
|
get(ui_api::connection_health),
|
||||||
)
|
)
|
||||||
.route("/ui/sites", get(ui_pages::sites_dashboard))
|
.route("/ui/sites", get(ui_pages::sites_dashboard))
|
||||||
|
.route("/ui/cluster", get(ui_pages::cluster_dashboard))
|
||||||
|
.route("/ui/cluster/data", get(ui_pages::cluster_data_json))
|
||||||
.route("/ui/sites/local", post(ui_pages::update_local_site))
|
.route("/ui/sites/local", post(ui_pages::update_local_site))
|
||||||
.route("/ui/sites/peers", post(ui_pages::add_peer_site))
|
.route("/ui/sites/peers", post(ui_pages::add_peer_site))
|
||||||
.route(
|
.route(
|
||||||
@@ -304,20 +315,30 @@ pub fn create_ui_router(state: state::AppState) -> Router {
|
|||||||
|
|
||||||
let public = Router::new()
|
let public = Router::new()
|
||||||
.route("/login", get(ui::login_page).post(ui::login_submit))
|
.route("/login", get(ui::login_page).post(ui::login_submit))
|
||||||
.route("/logout", post(ui::logout).get(ui::logout))
|
.route("/logout", post(ui::logout).get(ui::logout));
|
||||||
.route("/csrf-error", get(ui::csrf_error_page));
|
|
||||||
|
|
||||||
let session_state = middleware::SessionLayerState {
|
let session_state = middleware::SessionLayerState {
|
||||||
store: state.sessions.clone(),
|
store: state.sessions.clone(),
|
||||||
secure: false,
|
secure: false,
|
||||||
|
ttl: std::time::Duration::from_secs(
|
||||||
|
state.config.session_lifetime_days.saturating_mul(86_400),
|
||||||
|
),
|
||||||
};
|
};
|
||||||
|
|
||||||
let static_service = tower_http::services::ServeDir::new(&state.config.static_dir);
|
let static_router = Router::new()
|
||||||
|
.route(
|
||||||
|
"/static/{*path}",
|
||||||
|
axum::routing::get(handlers::static_assets::serve),
|
||||||
|
)
|
||||||
|
.with_state(state.clone());
|
||||||
|
|
||||||
protected
|
protected
|
||||||
.merge(public)
|
.merge(public)
|
||||||
.fallback(ui::not_found_page)
|
.fallback(ui::not_found_page)
|
||||||
.layer(axum::middleware::from_fn(middleware::csrf_layer))
|
.layer(axum::middleware::from_fn_with_state(
|
||||||
|
state.clone(),
|
||||||
|
middleware::csrf_layer,
|
||||||
|
))
|
||||||
.layer(axum::middleware::from_fn_with_state(
|
.layer(axum::middleware::from_fn_with_state(
|
||||||
session_state,
|
session_state,
|
||||||
middleware::session_layer,
|
middleware::session_layer,
|
||||||
@@ -327,14 +348,18 @@ pub fn create_ui_router(state: state::AppState) -> Router {
|
|||||||
middleware::ui_metrics_layer,
|
middleware::ui_metrics_layer,
|
||||||
))
|
))
|
||||||
.with_state(state)
|
.with_state(state)
|
||||||
.nest_service("/static", static_service)
|
.merge(static_router)
|
||||||
.layer(axum::middleware::from_fn(middleware::server_header))
|
.layer(axum::middleware::from_fn(middleware::server_header))
|
||||||
.layer(tower_http::compression::CompressionLayer::new())
|
.layer(tower_http::compression::CompressionLayer::new())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn create_router(state: state::AppState) -> Router {
|
pub fn create_router(state: state::AppState) -> Router {
|
||||||
let default_rate_limit = middleware::RateLimitLayerState::new(
|
let default_rate_limit = middleware::RateLimitLayerState::with_per_op(
|
||||||
state.config.ratelimit_default,
|
state.config.ratelimit_default,
|
||||||
|
state.config.ratelimit_list_buckets,
|
||||||
|
state.config.ratelimit_bucket_ops,
|
||||||
|
state.config.ratelimit_object_ops,
|
||||||
|
state.config.ratelimit_head_ops,
|
||||||
state.config.num_trusted_proxies,
|
state.config.num_trusted_proxies,
|
||||||
);
|
);
|
||||||
let admin_rate_limit = middleware::RateLimitLayerState::new(
|
let admin_rate_limit = middleware::RateLimitLayerState::new(
|
||||||
@@ -460,6 +485,14 @@ pub fn create_router(state: state::AppState) -> Router {
|
|||||||
"/admin/sites/{site_id}/bidirectional-status",
|
"/admin/sites/{site_id}/bidirectional-status",
|
||||||
axum::routing::get(handlers::admin::check_bidirectional_status),
|
axum::routing::get(handlers::admin::check_bidirectional_status),
|
||||||
)
|
)
|
||||||
|
.route(
|
||||||
|
"/admin/sync/stats",
|
||||||
|
axum::routing::get(handlers::admin::get_sync_stats),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/admin/cluster/overview",
|
||||||
|
axum::routing::get(handlers::admin::get_cluster_overview),
|
||||||
|
)
|
||||||
.route(
|
.route(
|
||||||
"/admin/topology",
|
"/admin/topology",
|
||||||
axum::routing::get(handlers::admin::get_topology),
|
axum::routing::get(handlers::admin::get_topology),
|
||||||
@@ -575,11 +608,22 @@ pub fn create_router(state: state::AppState) -> Router {
|
|||||||
middleware::rate_limit_layer,
|
middleware::rate_limit_layer,
|
||||||
));
|
));
|
||||||
|
|
||||||
|
let request_body_timeout =
|
||||||
|
std::time::Duration::from_secs(state.config.request_body_timeout_secs);
|
||||||
|
|
||||||
api_router
|
api_router
|
||||||
.merge(admin_router)
|
.merge(admin_router)
|
||||||
.layer(axum::middleware::from_fn(middleware::server_header))
|
.layer(axum::middleware::from_fn(middleware::server_header))
|
||||||
.layer(cors_layer(&state.config))
|
.layer(cors_layer(&state.config))
|
||||||
|
.layer(axum::middleware::from_fn_with_state(
|
||||||
|
state.clone(),
|
||||||
|
middleware::bucket_cors_layer,
|
||||||
|
))
|
||||||
|
.layer(axum::middleware::from_fn(middleware::request_log_layer))
|
||||||
.layer(tower_http::compression::CompressionLayer::new())
|
.layer(tower_http::compression::CompressionLayer::new())
|
||||||
|
.layer(tower_http::timeout::RequestBodyTimeoutLayer::new(
|
||||||
|
request_body_timeout,
|
||||||
|
))
|
||||||
.with_state(state)
|
.with_state(state)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -189,6 +189,11 @@ async fn main() {
|
|||||||
|
|
||||||
let shutdown = shutdown_signal_shared();
|
let shutdown = shutdown_signal_shared();
|
||||||
let api_shutdown = shutdown.clone();
|
let api_shutdown = shutdown.clone();
|
||||||
|
let api_listener = axum::serve::ListenerExt::tap_io(api_listener, |stream| {
|
||||||
|
if let Err(err) = stream.set_nodelay(true) {
|
||||||
|
tracing::trace!("failed to set TCP_NODELAY on api socket: {}", err);
|
||||||
|
}
|
||||||
|
});
|
||||||
let api_task = tokio::spawn(async move {
|
let api_task = tokio::spawn(async move {
|
||||||
axum::serve(
|
axum::serve(
|
||||||
api_listener,
|
api_listener,
|
||||||
@@ -202,6 +207,11 @@ async fn main() {
|
|||||||
|
|
||||||
let ui_task = if let (Some(listener), Some(app)) = (ui_listener, ui_app) {
|
let ui_task = if let (Some(listener), Some(app)) = (ui_listener, ui_app) {
|
||||||
let ui_shutdown = shutdown.clone();
|
let ui_shutdown = shutdown.clone();
|
||||||
|
let listener = axum::serve::ListenerExt::tap_io(listener, |stream| {
|
||||||
|
if let Err(err) = stream.set_nodelay(true) {
|
||||||
|
tracing::trace!("failed to set TCP_NODELAY on ui socket: {}", err);
|
||||||
|
}
|
||||||
|
});
|
||||||
Some(tokio::spawn(async move {
|
Some(tokio::spawn(async move {
|
||||||
axum::serve(listener, app)
|
axum::serve(listener, app)
|
||||||
.with_graceful_shutdown(async move {
|
.with_graceful_shutdown(async move {
|
||||||
@@ -465,13 +475,17 @@ fn ensure_iam_bootstrap(config: &ServerConfig) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
tracing::info!("============================================================");
|
println!("============================================================");
|
||||||
tracing::info!("MYFSIO - ADMIN CREDENTIALS INITIALIZED");
|
println!("MYFSIO - ADMIN CREDENTIALS INITIALIZED");
|
||||||
tracing::info!("============================================================");
|
println!("============================================================");
|
||||||
tracing::info!("Access Key: {}", access_key);
|
println!("Access Key: {}", access_key);
|
||||||
tracing::info!("Secret Key: {}", secret_key);
|
println!("Secret Key: {}", secret_key);
|
||||||
tracing::info!("Saved to: {}", iam_path.display());
|
println!("Saved to: {}", iam_path.display());
|
||||||
tracing::info!("============================================================");
|
println!("============================================================");
|
||||||
|
tracing::info!(
|
||||||
|
"Admin credentials initialized; access key written to {}",
|
||||||
|
iam_path.display()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn reset_admin_credentials(config: &ServerConfig) {
|
fn reset_admin_credentials(config: &ServerConfig) {
|
||||||
|
|||||||
@@ -12,9 +12,74 @@ use serde_json::Value;
|
|||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
use tokio::io::AsyncReadExt;
|
use tokio::io::AsyncReadExt;
|
||||||
|
|
||||||
|
use crate::middleware::sha_body::{is_hex_sha256, Sha256VerifyBody};
|
||||||
use crate::services::acl::acl_from_bucket_config;
|
use crate::services::acl::acl_from_bucket_config;
|
||||||
use crate::state::AppState;
|
use crate::state::AppState;
|
||||||
|
|
||||||
|
fn wrap_body_for_sha256_verification(req: &mut Request) -> Option<Response> {
|
||||||
|
let declared = match req
|
||||||
|
.headers()
|
||||||
|
.get("x-amz-content-sha256")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
{
|
||||||
|
Some(v) => v.to_string(),
|
||||||
|
None => return None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let upper = declared.to_ascii_uppercase();
|
||||||
|
let is_streaming_signed = upper == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
|
||||||
|
|| upper == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER";
|
||||||
|
let is_streaming_unsigned = upper == "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
|
||||||
|
let is_streaming = is_streaming_signed || is_streaming_unsigned;
|
||||||
|
|
||||||
|
if is_streaming {
|
||||||
|
if std::env::var("STRICT_STREAMING_SIGV4")
|
||||||
|
.ok()
|
||||||
|
.as_deref()
|
||||||
|
.map(|v| v.eq_ignore_ascii_case("true") || v == "1")
|
||||||
|
.unwrap_or(false)
|
||||||
|
{
|
||||||
|
tracing::warn!(
|
||||||
|
payload_type = %upper,
|
||||||
|
"Rejecting streaming SigV4 request because STRICT_STREAMING_SIGV4 is enabled"
|
||||||
|
);
|
||||||
|
let err = S3Error::new(
|
||||||
|
S3ErrorCode::SignatureDoesNotMatch,
|
||||||
|
"Streaming SigV4 chunk-signature validation is not yet implemented; \
|
||||||
|
resend with x-amz-content-sha256: UNSIGNED-PAYLOAD or disable STRICT_STREAMING_SIGV4",
|
||||||
|
);
|
||||||
|
let status = StatusCode::from_u16(err.http_status())
|
||||||
|
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
||||||
|
let code_str = err.code.as_str();
|
||||||
|
return Some(
|
||||||
|
(
|
||||||
|
status,
|
||||||
|
[
|
||||||
|
("content-type", "application/xml"),
|
||||||
|
("x-amz-error-code", code_str),
|
||||||
|
],
|
||||||
|
err.to_xml(),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
tracing::warn!(
|
||||||
|
payload_type = %upper,
|
||||||
|
"Accepting streaming SigV4 request without per-chunk signature validation. \
|
||||||
|
Set STRICT_STREAMING_SIGV4=true to reject these requests until full validation lands."
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
if !is_hex_sha256(&declared) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let body = std::mem::replace(req.body_mut(), axum::body::Body::empty());
|
||||||
|
let wrapped = Sha256VerifyBody::new(body, declared);
|
||||||
|
*req.body_mut() = axum::body::Body::new(wrapped);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
struct OriginalCanonicalPath(String);
|
struct OriginalCanonicalPath(String);
|
||||||
|
|
||||||
@@ -135,6 +200,39 @@ fn parse_website_config(value: &Value) -> Option<(String, Option<String>)> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn apply_website_object_headers(
|
||||||
|
headers: &mut HeaderMap,
|
||||||
|
meta: &myfsio_common::types::ObjectMeta,
|
||||||
|
) {
|
||||||
|
if let Some(ref etag) = meta.etag {
|
||||||
|
if let Ok(value) = format!("\"{}\"", etag).parse() {
|
||||||
|
headers.insert(header::ETAG, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Ok(value) = meta
|
||||||
|
.last_modified
|
||||||
|
.format("%a, %d %b %Y %H:%M:%S GMT")
|
||||||
|
.to_string()
|
||||||
|
.parse()
|
||||||
|
{
|
||||||
|
headers.insert(header::LAST_MODIFIED, value);
|
||||||
|
}
|
||||||
|
if let Some(enc_info) =
|
||||||
|
myfsio_crypto::encryption::EncryptionMetadata::from_metadata(&meta.internal_metadata)
|
||||||
|
{
|
||||||
|
if let Ok(value) = enc_info.algorithm.as_str().parse() {
|
||||||
|
headers.insert("x-amz-server-side-encryption", value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (k, v) in &meta.metadata {
|
||||||
|
if let Ok(header_val) = v.parse() {
|
||||||
|
if let Ok(name) = format!("x-amz-meta-{}", k).parse::<axum::http::HeaderName>() {
|
||||||
|
headers.insert(name, header_val);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn serve_website_document(
|
async fn serve_website_document(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -155,6 +253,7 @@ async fn serve_website_document(
|
|||||||
meta.size.to_string().parse().unwrap(),
|
meta.size.to_string().parse().unwrap(),
|
||||||
);
|
);
|
||||||
headers.insert(header::ACCEPT_RANGES, "bytes".parse().unwrap());
|
headers.insert(header::ACCEPT_RANGES, "bytes".parse().unwrap());
|
||||||
|
apply_website_object_headers(&mut headers, &meta);
|
||||||
return Some((status, headers).into_response());
|
return Some((status, headers).into_response());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -166,6 +265,7 @@ async fn serve_website_document(
|
|||||||
let mut headers = HeaderMap::new();
|
let mut headers = HeaderMap::new();
|
||||||
headers.insert(header::CONTENT_TYPE, content_type.parse().unwrap());
|
headers.insert(header::CONTENT_TYPE, content_type.parse().unwrap());
|
||||||
headers.insert(header::ACCEPT_RANGES, "bytes".parse().unwrap());
|
headers.insert(header::ACCEPT_RANGES, "bytes".parse().unwrap());
|
||||||
|
apply_website_object_headers(&mut headers, &meta);
|
||||||
|
|
||||||
if status == StatusCode::OK {
|
if status == StatusCode::OK {
|
||||||
if let Some(range_header) = range_header {
|
if let Some(range_header) = range_header {
|
||||||
@@ -474,8 +574,18 @@ pub async fn auth_layer(State(state): State<AppState>, mut req: Request, next: N
|
|||||||
{
|
{
|
||||||
error_response(err, &auth_path)
|
error_response(err, &auth_path)
|
||||||
} else {
|
} else {
|
||||||
|
if let Some(registry) = state.site_registry.as_ref() {
|
||||||
|
if registry.is_peer_inbound_access_key(&principal.access_key) {
|
||||||
|
req.extensions_mut()
|
||||||
|
.insert(crate::middleware::ReplicationPeerRequest);
|
||||||
|
}
|
||||||
|
}
|
||||||
req.extensions_mut().insert(principal);
|
req.extensions_mut().insert(principal);
|
||||||
next.run(req).await
|
if let Some(rejection) = wrap_body_for_sha256_verification(&mut req) {
|
||||||
|
rejection
|
||||||
|
} else {
|
||||||
|
next.run(req).await
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
AuthResult::Denied(err) => error_response(err, &auth_path),
|
AuthResult::Denied(err) => error_response(err, &auth_path),
|
||||||
@@ -1102,7 +1212,9 @@ fn verify_sigv4_header(state: &AppState, req: &Request, auth_str: &str) -> AuthR
|
|||||||
let parts: Vec<&str> = auth_str
|
let parts: Vec<&str> = auth_str
|
||||||
.strip_prefix("AWS4-HMAC-SHA256 ")
|
.strip_prefix("AWS4-HMAC-SHA256 ")
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.split(", ")
|
.split(',')
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
if parts.len() != 3 {
|
if parts.len() != 3 {
|
||||||
@@ -1112,9 +1224,24 @@ fn verify_sigv4_header(state: &AppState, req: &Request, auth_str: &str) -> AuthR
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
let credential = parts[0].strip_prefix("Credential=").unwrap_or("");
|
let mut credential: &str = "";
|
||||||
let signed_headers_str = parts[1].strip_prefix("SignedHeaders=").unwrap_or("");
|
let mut signed_headers_str: &str = "";
|
||||||
let provided_signature = parts[2].strip_prefix("Signature=").unwrap_or("");
|
let mut provided_signature: &str = "";
|
||||||
|
for part in &parts {
|
||||||
|
if let Some(v) = part.strip_prefix("Credential=") {
|
||||||
|
credential = v;
|
||||||
|
} else if let Some(v) = part.strip_prefix("SignedHeaders=") {
|
||||||
|
signed_headers_str = v;
|
||||||
|
} else if let Some(v) = part.strip_prefix("Signature=") {
|
||||||
|
provided_signature = v;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if credential.is_empty() || signed_headers_str.is_empty() || provided_signature.is_empty() {
|
||||||
|
return AuthResult::Denied(S3Error::new(
|
||||||
|
S3ErrorCode::InvalidArgument,
|
||||||
|
"Malformed Authorization header",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
let cred_parts: Vec<&str> = credential.split('/').collect();
|
let cred_parts: Vec<&str> = credential.split('/').collect();
|
||||||
if cred_parts.len() != 5 {
|
if cred_parts.len() != 5 {
|
||||||
@@ -1299,7 +1426,7 @@ fn verify_sigv4_query(state: &AppState, req: &Request) -> AuthResult {
|
|||||||
}
|
}
|
||||||
if elapsed < -(state.config.sigv4_timestamp_tolerance_secs as i64) {
|
if elapsed < -(state.config.sigv4_timestamp_tolerance_secs as i64) {
|
||||||
return AuthResult::Denied(S3Error::new(
|
return AuthResult::Denied(S3Error::new(
|
||||||
S3ErrorCode::AccessDenied,
|
S3ErrorCode::RequestTimeTooSkewed,
|
||||||
"Request is too far in the future",
|
"Request is too far in the future",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
@@ -1369,8 +1496,11 @@ fn check_timestamp_freshness(amz_date: &str, tolerance_secs: u64) -> Option<S3Er
|
|||||||
|
|
||||||
if diff > tolerance_secs {
|
if diff > tolerance_secs {
|
||||||
return Some(S3Error::new(
|
return Some(S3Error::new(
|
||||||
S3ErrorCode::AccessDenied,
|
S3ErrorCode::RequestTimeTooSkewed,
|
||||||
"Request timestamp too old or too far in the future",
|
format!(
|
||||||
|
"The difference between the request time and the server's time is too large ({}s, tolerance {}s)",
|
||||||
|
diff, tolerance_secs
|
||||||
|
),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
None
|
None
|
||||||
@@ -1401,9 +1531,18 @@ fn error_response(err: S3Error, resource: &str) -> Response {
|
|||||||
let status =
|
let status =
|
||||||
StatusCode::from_u16(err.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
StatusCode::from_u16(err.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
||||||
let request_id = uuid::Uuid::new_v4().simple().to_string();
|
let request_id = uuid::Uuid::new_v4().simple().to_string();
|
||||||
|
let code_str = err.code.as_str();
|
||||||
let body = err
|
let body = err
|
||||||
.with_resource(resource.to_string())
|
.with_resource(resource.to_string())
|
||||||
.with_request_id(request_id)
|
.with_request_id(request_id)
|
||||||
.to_xml();
|
.to_xml();
|
||||||
(status, [("content-type", "application/xml")], body).into_response()
|
(
|
||||||
|
status,
|
||||||
|
[
|
||||||
|
("content-type", "application/xml"),
|
||||||
|
("x-amz-error-code", code_str),
|
||||||
|
],
|
||||||
|
body,
|
||||||
|
)
|
||||||
|
.into_response()
|
||||||
}
|
}
|
||||||
|
|||||||
276
crates/myfsio-server/src/middleware/bucket_cors.rs
Normal file
276
crates/myfsio-server/src/middleware/bucket_cors.rs
Normal file
@@ -0,0 +1,276 @@
|
|||||||
|
use axum::extract::{Request, State};
|
||||||
|
use axum::http::{HeaderMap, HeaderValue, Method, StatusCode};
|
||||||
|
use axum::middleware::Next;
|
||||||
|
use axum::response::{IntoResponse, Response};
|
||||||
|
use myfsio_storage::traits::StorageEngine;
|
||||||
|
|
||||||
|
use crate::state::AppState;
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Clone)]
|
||||||
|
struct CorsRule {
|
||||||
|
allowed_origins: Vec<String>,
|
||||||
|
allowed_methods: Vec<String>,
|
||||||
|
allowed_headers: Vec<String>,
|
||||||
|
expose_headers: Vec<String>,
|
||||||
|
max_age_seconds: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_cors_config(xml: &str) -> Vec<CorsRule> {
|
||||||
|
let doc = match roxmltree::Document::parse(xml) {
|
||||||
|
Ok(d) => d,
|
||||||
|
Err(_) => return Vec::new(),
|
||||||
|
};
|
||||||
|
let mut rules = Vec::new();
|
||||||
|
for rule_node in doc
|
||||||
|
.descendants()
|
||||||
|
.filter(|node| node.is_element() && node.tag_name().name() == "CORSRule")
|
||||||
|
{
|
||||||
|
let mut rule = CorsRule::default();
|
||||||
|
for child in rule_node.children().filter(|n| n.is_element()) {
|
||||||
|
let text = child.text().unwrap_or("").trim().to_string();
|
||||||
|
match child.tag_name().name() {
|
||||||
|
"AllowedOrigin" => rule.allowed_origins.push(text),
|
||||||
|
"AllowedMethod" => rule.allowed_methods.push(text.to_ascii_uppercase()),
|
||||||
|
"AllowedHeader" => rule.allowed_headers.push(text),
|
||||||
|
"ExposeHeader" => rule.expose_headers.push(text),
|
||||||
|
"MaxAgeSeconds" => {
|
||||||
|
if let Ok(v) = text.parse::<u64>() {
|
||||||
|
rule.max_age_seconds = Some(v);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rules.push(rule);
|
||||||
|
}
|
||||||
|
rules
|
||||||
|
}
|
||||||
|
|
||||||
|
fn match_origin(pattern: &str, origin: &str) -> bool {
|
||||||
|
if pattern == "*" {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if pattern == origin {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if let Some(suffix) = pattern.strip_prefix('*') {
|
||||||
|
return origin.ends_with(suffix);
|
||||||
|
}
|
||||||
|
if let Some(prefix) = pattern.strip_suffix('*') {
|
||||||
|
return origin.starts_with(prefix);
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
fn match_header(pattern: &str, header: &str) -> bool {
|
||||||
|
if pattern == "*" {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
pattern.eq_ignore_ascii_case(header)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn find_matching_rule<'a>(
|
||||||
|
rules: &'a [CorsRule],
|
||||||
|
origin: &str,
|
||||||
|
method: &str,
|
||||||
|
request_headers: &[&str],
|
||||||
|
) -> Option<&'a CorsRule> {
|
||||||
|
rules.iter().find(|rule| {
|
||||||
|
let origin_match = rule.allowed_origins.iter().any(|p| match_origin(p, origin));
|
||||||
|
if !origin_match {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let method_match = rule
|
||||||
|
.allowed_methods
|
||||||
|
.iter()
|
||||||
|
.any(|m| m.eq_ignore_ascii_case(method));
|
||||||
|
if !method_match {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
request_headers.iter().all(|h| {
|
||||||
|
rule.allowed_headers
|
||||||
|
.iter()
|
||||||
|
.any(|pattern| match_header(pattern, h))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn find_matching_rule_for_actual<'a>(
|
||||||
|
rules: &'a [CorsRule],
|
||||||
|
origin: &str,
|
||||||
|
method: &str,
|
||||||
|
) -> Option<&'a CorsRule> {
|
||||||
|
rules.iter().find(|rule| {
|
||||||
|
rule.allowed_origins.iter().any(|p| match_origin(p, origin))
|
||||||
|
&& rule
|
||||||
|
.allowed_methods
|
||||||
|
.iter()
|
||||||
|
.any(|m| m.eq_ignore_ascii_case(method))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bucket_from_path(path: &str) -> Option<&str> {
|
||||||
|
let trimmed = path.trim_start_matches('/');
|
||||||
|
if trimmed.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if trimmed.starts_with("admin/")
|
||||||
|
|| trimmed.starts_with("myfsio/")
|
||||||
|
|| trimmed.starts_with("kms/")
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let first = trimmed.split('/').next().unwrap_or("");
|
||||||
|
if myfsio_storage::validation::validate_bucket_name(first).is_some() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(first)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn bucket_from_host(state: &AppState, headers: &HeaderMap) -> Option<String> {
|
||||||
|
let host = headers
|
||||||
|
.get("host")
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.and_then(|value| value.split(':').next())?
|
||||||
|
.trim()
|
||||||
|
.to_ascii_lowercase();
|
||||||
|
let (candidate, _) = host.split_once('.')?;
|
||||||
|
if myfsio_storage::validation::validate_bucket_name(candidate).is_some() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
match state.storage.bucket_exists(candidate).await {
|
||||||
|
Ok(true) => Some(candidate.to_string()),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn resolve_bucket(state: &AppState, headers: &HeaderMap, path: &str) -> Option<String> {
|
||||||
|
if let Some(name) = bucket_from_host(state, headers).await {
|
||||||
|
return Some(name);
|
||||||
|
}
|
||||||
|
bucket_from_path(path).map(str::to_string)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_rule_headers(headers: &mut axum::http::HeaderMap, rule: &CorsRule, origin: &str) {
|
||||||
|
headers.remove("access-control-allow-origin");
|
||||||
|
headers.remove("vary");
|
||||||
|
if let Ok(val) = HeaderValue::from_str(origin) {
|
||||||
|
headers.insert("access-control-allow-origin", val);
|
||||||
|
}
|
||||||
|
headers.insert("vary", HeaderValue::from_static("Origin"));
|
||||||
|
if !rule.expose_headers.is_empty() {
|
||||||
|
let value = rule.expose_headers.join(", ");
|
||||||
|
if let Ok(val) = HeaderValue::from_str(&value) {
|
||||||
|
headers.remove("access-control-expose-headers");
|
||||||
|
headers.insert("access-control-expose-headers", val);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn strip_cors_response_headers(headers: &mut HeaderMap) {
|
||||||
|
headers.remove("access-control-allow-origin");
|
||||||
|
headers.remove("access-control-allow-credentials");
|
||||||
|
headers.remove("access-control-expose-headers");
|
||||||
|
headers.remove("access-control-allow-methods");
|
||||||
|
headers.remove("access-control-allow-headers");
|
||||||
|
headers.remove("access-control-max-age");
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn bucket_cors_layer(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
req: Request,
|
||||||
|
next: Next,
|
||||||
|
) -> Response {
|
||||||
|
let path = req.uri().path().to_string();
|
||||||
|
let bucket = match resolve_bucket(&state, req.headers(), &path).await {
|
||||||
|
Some(name) => name,
|
||||||
|
None => return next.run(req).await,
|
||||||
|
};
|
||||||
|
|
||||||
|
let origin = req
|
||||||
|
.headers()
|
||||||
|
.get("origin")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
|
||||||
|
let bucket_rules = if origin.is_some() {
|
||||||
|
match state.storage.get_bucket_config(&bucket).await {
|
||||||
|
Ok(cfg) => cfg
|
||||||
|
.cors
|
||||||
|
.as_ref()
|
||||||
|
.map(|v| match v {
|
||||||
|
serde_json::Value::String(s) => s.clone(),
|
||||||
|
other => other.to_string(),
|
||||||
|
})
|
||||||
|
.map(|xml| parse_cors_config(&xml))
|
||||||
|
.filter(|rules| !rules.is_empty()),
|
||||||
|
Err(_) => None,
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
let is_preflight = req.method() == Method::OPTIONS
|
||||||
|
&& req.headers().contains_key("access-control-request-method");
|
||||||
|
|
||||||
|
if is_preflight {
|
||||||
|
if let (Some(origin), Some(rules)) = (origin.as_deref(), bucket_rules.as_ref()) {
|
||||||
|
let req_method = req
|
||||||
|
.headers()
|
||||||
|
.get("access-control-request-method")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.unwrap_or("");
|
||||||
|
let req_headers_raw = req
|
||||||
|
.headers()
|
||||||
|
.get("access-control-request-headers")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.unwrap_or("");
|
||||||
|
let req_headers: Vec<&str> = req_headers_raw
|
||||||
|
.split(',')
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
if let Some(rule) = find_matching_rule(rules, origin, req_method, &req_headers) {
|
||||||
|
let mut resp = StatusCode::NO_CONTENT.into_response();
|
||||||
|
apply_rule_headers(resp.headers_mut(), rule, origin);
|
||||||
|
let methods_value = rule.allowed_methods.join(", ");
|
||||||
|
if let Ok(val) = HeaderValue::from_str(&methods_value) {
|
||||||
|
resp.headers_mut()
|
||||||
|
.insert("access-control-allow-methods", val);
|
||||||
|
}
|
||||||
|
let headers_value = if rule.allowed_headers.iter().any(|h| h == "*") {
|
||||||
|
req_headers_raw.to_string()
|
||||||
|
} else {
|
||||||
|
rule.allowed_headers.join(", ")
|
||||||
|
};
|
||||||
|
if !headers_value.is_empty() {
|
||||||
|
if let Ok(val) = HeaderValue::from_str(&headers_value) {
|
||||||
|
resp.headers_mut()
|
||||||
|
.insert("access-control-allow-headers", val);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(max_age) = rule.max_age_seconds {
|
||||||
|
if let Ok(val) = HeaderValue::from_str(&max_age.to_string()) {
|
||||||
|
resp.headers_mut().insert("access-control-max-age", val);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return resp;
|
||||||
|
}
|
||||||
|
return (StatusCode::FORBIDDEN, "CORSResponse: CORS is not enabled").into_response();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let method = req.method().clone();
|
||||||
|
let mut resp = next.run(req).await;
|
||||||
|
|
||||||
|
if let (Some(origin), Some(rules)) = (origin.as_deref(), bucket_rules.as_ref()) {
|
||||||
|
if let Some(rule) = find_matching_rule_for_actual(rules, origin, method.as_str()) {
|
||||||
|
apply_rule_headers(resp.headers_mut(), rule, origin);
|
||||||
|
} else {
|
||||||
|
strip_cors_response_headers(resp.headers_mut());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resp
|
||||||
|
}
|
||||||
@@ -1,11 +1,17 @@
|
|||||||
mod auth;
|
mod auth;
|
||||||
|
mod bucket_cors;
|
||||||
pub mod ratelimit;
|
pub mod ratelimit;
|
||||||
pub mod session;
|
pub mod session;
|
||||||
|
pub(crate) mod sha_body;
|
||||||
|
|
||||||
pub use auth::auth_layer;
|
pub use auth::auth_layer;
|
||||||
|
pub use bucket_cors::bucket_cors_layer;
|
||||||
pub use ratelimit::{rate_limit_layer, RateLimitLayerState};
|
pub use ratelimit::{rate_limit_layer, RateLimitLayerState};
|
||||||
pub use session::{csrf_layer, session_layer, SessionHandle, SessionLayerState};
|
pub use session::{csrf_layer, session_layer, SessionHandle, SessionLayerState};
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug)]
|
||||||
|
pub struct ReplicationPeerRequest;
|
||||||
|
|
||||||
use axum::extract::{Request, State};
|
use axum::extract::{Request, State};
|
||||||
use axum::middleware::Next;
|
use axum::middleware::Next;
|
||||||
use axum::response::Response;
|
use axum::response::Response;
|
||||||
@@ -20,6 +26,42 @@ pub async fn server_header(req: Request, next: Next) -> Response {
|
|||||||
resp
|
resp
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn request_log_layer(req: Request, next: Next) -> Response {
|
||||||
|
let start = Instant::now();
|
||||||
|
let method = req.method().clone();
|
||||||
|
let uri = req.uri().clone();
|
||||||
|
let version = req.version();
|
||||||
|
let remote = req
|
||||||
|
.extensions()
|
||||||
|
.get::<axum::extract::ConnectInfo<std::net::SocketAddr>>()
|
||||||
|
.map(|ci| ci.0.ip().to_string())
|
||||||
|
.unwrap_or_else(|| "-".to_string());
|
||||||
|
|
||||||
|
let response = next.run(req).await;
|
||||||
|
|
||||||
|
let status = response.status().as_u16();
|
||||||
|
let elapsed_ms = start.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
let bytes_out = response
|
||||||
|
.headers()
|
||||||
|
.get(axum::http::header::CONTENT_LENGTH)
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.and_then(|v| v.parse::<u64>().ok());
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
target: "myfsio::access",
|
||||||
|
remote = %remote,
|
||||||
|
method = %method,
|
||||||
|
uri = %uri,
|
||||||
|
version = ?version,
|
||||||
|
status,
|
||||||
|
bytes_out = bytes_out.unwrap_or(0),
|
||||||
|
elapsed_ms = format!("{:.3}", elapsed_ms),
|
||||||
|
"request"
|
||||||
|
);
|
||||||
|
|
||||||
|
response
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn ui_metrics_layer(State(state): State<AppState>, req: Request, next: Next) -> Response {
|
pub async fn ui_metrics_layer(State(state): State<AppState>, req: Request, next: Next) -> Response {
|
||||||
let metrics = match state.metrics.clone() {
|
let metrics = match state.metrics.clone() {
|
||||||
Some(m) => m,
|
Some(m) => m,
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ use std::sync::Arc;
|
|||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use axum::extract::{ConnectInfo, Request, State};
|
use axum::extract::{ConnectInfo, Request, State};
|
||||||
use axum::http::{header, StatusCode};
|
use axum::http::{header, Method, StatusCode};
|
||||||
use axum::middleware::Next;
|
use axum::middleware::Next;
|
||||||
use axum::response::{IntoResponse, Response};
|
use axum::response::{IntoResponse, Response};
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
@@ -13,17 +13,77 @@ use crate::config::RateLimitSetting;
|
|||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct RateLimitLayerState {
|
pub struct RateLimitLayerState {
|
||||||
limiter: Arc<FixedWindowLimiter>,
|
default_limiter: Arc<FixedWindowLimiter>,
|
||||||
|
list_buckets_limiter: Option<Arc<FixedWindowLimiter>>,
|
||||||
|
bucket_ops_limiter: Option<Arc<FixedWindowLimiter>>,
|
||||||
|
object_ops_limiter: Option<Arc<FixedWindowLimiter>>,
|
||||||
|
head_ops_limiter: Option<Arc<FixedWindowLimiter>>,
|
||||||
num_trusted_proxies: usize,
|
num_trusted_proxies: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RateLimitLayerState {
|
impl RateLimitLayerState {
|
||||||
pub fn new(setting: RateLimitSetting, num_trusted_proxies: usize) -> Self {
|
pub fn new(setting: RateLimitSetting, num_trusted_proxies: usize) -> Self {
|
||||||
Self {
|
Self {
|
||||||
limiter: Arc::new(FixedWindowLimiter::new(setting)),
|
default_limiter: Arc::new(FixedWindowLimiter::new(setting)),
|
||||||
|
list_buckets_limiter: None,
|
||||||
|
bucket_ops_limiter: None,
|
||||||
|
object_ops_limiter: None,
|
||||||
|
head_ops_limiter: None,
|
||||||
num_trusted_proxies,
|
num_trusted_proxies,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn with_per_op(
|
||||||
|
default: RateLimitSetting,
|
||||||
|
list_buckets: RateLimitSetting,
|
||||||
|
bucket_ops: RateLimitSetting,
|
||||||
|
object_ops: RateLimitSetting,
|
||||||
|
head_ops: RateLimitSetting,
|
||||||
|
num_trusted_proxies: usize,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
default_limiter: Arc::new(FixedWindowLimiter::new(default)),
|
||||||
|
list_buckets_limiter: (list_buckets != default)
|
||||||
|
.then(|| Arc::new(FixedWindowLimiter::new(list_buckets))),
|
||||||
|
bucket_ops_limiter: (bucket_ops != default)
|
||||||
|
.then(|| Arc::new(FixedWindowLimiter::new(bucket_ops))),
|
||||||
|
object_ops_limiter: (object_ops != default)
|
||||||
|
.then(|| Arc::new(FixedWindowLimiter::new(object_ops))),
|
||||||
|
head_ops_limiter: (head_ops != default)
|
||||||
|
.then(|| Arc::new(FixedWindowLimiter::new(head_ops))),
|
||||||
|
num_trusted_proxies,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn select_limiter(&self, req: &Request) -> &Arc<FixedWindowLimiter> {
|
||||||
|
let path = req.uri().path();
|
||||||
|
let method = req.method();
|
||||||
|
if path == "/" && *method == Method::GET {
|
||||||
|
if let Some(ref limiter) = self.list_buckets_limiter {
|
||||||
|
return limiter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let segments: Vec<&str> = path
|
||||||
|
.trim_start_matches('/')
|
||||||
|
.split('/')
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.collect();
|
||||||
|
if *method == Method::HEAD {
|
||||||
|
if let Some(ref limiter) = self.head_ops_limiter {
|
||||||
|
return limiter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if segments.len() == 1 {
|
||||||
|
if let Some(ref limiter) = self.bucket_ops_limiter {
|
||||||
|
return limiter;
|
||||||
|
}
|
||||||
|
} else if segments.len() >= 2 {
|
||||||
|
if let Some(ref limiter) = self.object_ops_limiter {
|
||||||
|
return limiter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
&self.default_limiter
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@@ -99,22 +159,32 @@ pub async fn rate_limit_layer(
|
|||||||
next: Next,
|
next: Next,
|
||||||
) -> Response {
|
) -> Response {
|
||||||
let key = rate_limit_key(&req, state.num_trusted_proxies);
|
let key = rate_limit_key(&req, state.num_trusted_proxies);
|
||||||
match state.limiter.check(&key) {
|
let limiter = state.select_limiter(&req);
|
||||||
|
match limiter.check(&key) {
|
||||||
Ok(()) => next.run(req).await,
|
Ok(()) => next.run(req).await,
|
||||||
Err(retry_after) => too_many_requests(retry_after),
|
Err(retry_after) => {
|
||||||
|
let resource = req.uri().path().to_string();
|
||||||
|
too_many_requests(retry_after, &resource)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn too_many_requests(retry_after: u64) -> Response {
|
fn too_many_requests(retry_after: u64, resource: &str) -> Response {
|
||||||
(
|
let request_id = uuid::Uuid::new_v4().simple().to_string();
|
||||||
StatusCode::TOO_MANY_REQUESTS,
|
let body = myfsio_xml::response::rate_limit_exceeded_xml(resource, &request_id);
|
||||||
|
let mut response = (
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
[
|
[
|
||||||
(header::CONTENT_TYPE, "application/xml".to_string()),
|
(header::CONTENT_TYPE, "application/xml".to_string()),
|
||||||
(header::RETRY_AFTER, retry_after.to_string()),
|
(header::RETRY_AFTER, retry_after.to_string()),
|
||||||
],
|
],
|
||||||
myfsio_xml::response::rate_limit_exceeded_xml(),
|
body,
|
||||||
)
|
)
|
||||||
.into_response()
|
.into_response();
|
||||||
|
if let Ok(value) = request_id.parse() {
|
||||||
|
response.headers_mut().insert("x-amz-request-id", value);
|
||||||
|
}
|
||||||
|
response
|
||||||
}
|
}
|
||||||
|
|
||||||
fn rate_limit_key(req: &Request, num_trusted_proxies: usize) -> String {
|
fn rate_limit_key(req: &Request, num_trusted_proxies: usize) -> String {
|
||||||
|
|||||||
@@ -1,10 +1,11 @@
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use axum::extract::{Request, State};
|
use axum::extract::{Request, State};
|
||||||
use axum::http::{header, HeaderValue, StatusCode};
|
use axum::http::{header, HeaderValue, StatusCode};
|
||||||
use axum::middleware::Next;
|
use axum::middleware::Next;
|
||||||
use axum::response::{IntoResponse, Response};
|
use axum::response::{IntoResponse, Response};
|
||||||
use cookie::{Cookie, SameSite};
|
use cookie::{time::Duration as CookieDuration, Cookie, SameSite};
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
|
|
||||||
use crate::session::{
|
use crate::session::{
|
||||||
@@ -16,6 +17,7 @@ use crate::session::{
|
|||||||
pub struct SessionLayerState {
|
pub struct SessionLayerState {
|
||||||
pub store: Arc<SessionStore>,
|
pub store: Arc<SessionStore>,
|
||||||
pub secure: bool,
|
pub secure: bool,
|
||||||
|
pub ttl: Duration,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -23,6 +25,8 @@ pub struct SessionHandle {
|
|||||||
pub id: String,
|
pub id: String,
|
||||||
inner: Arc<Mutex<SessionData>>,
|
inner: Arc<Mutex<SessionData>>,
|
||||||
dirty: Arc<Mutex<bool>>,
|
dirty: Arc<Mutex<bool>>,
|
||||||
|
rotated_id: Arc<Mutex<Option<String>>>,
|
||||||
|
destroy_old: Arc<Mutex<Option<String>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SessionHandle {
|
impl SessionHandle {
|
||||||
@@ -31,6 +35,8 @@ impl SessionHandle {
|
|||||||
id,
|
id,
|
||||||
inner: Arc::new(Mutex::new(data)),
|
inner: Arc::new(Mutex::new(data)),
|
||||||
dirty: Arc::new(Mutex::new(false)),
|
dirty: Arc::new(Mutex::new(false)),
|
||||||
|
rotated_id: Arc::new(Mutex::new(None)),
|
||||||
|
destroy_old: Arc::new(Mutex::new(None)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -53,6 +59,21 @@ impl SessionHandle {
|
|||||||
pub fn is_dirty(&self) -> bool {
|
pub fn is_dirty(&self) -> bool {
|
||||||
*self.dirty.lock()
|
*self.dirty.lock()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn rotate_id(&self) {
|
||||||
|
let new_id = crate::session::generate_token(32);
|
||||||
|
*self.destroy_old.lock() = Some(self.id.clone());
|
||||||
|
*self.rotated_id.lock() = Some(new_id);
|
||||||
|
*self.dirty.lock() = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn take_rotated_id(&self) -> Option<String> {
|
||||||
|
self.rotated_id.lock().take()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn take_destroy_old(&self) -> Option<String> {
|
||||||
|
self.destroy_old.lock().take()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn session_layer(
|
pub async fn session_layer(
|
||||||
@@ -62,13 +83,10 @@ pub async fn session_layer(
|
|||||||
) -> Response {
|
) -> Response {
|
||||||
let cookie_id = extract_session_cookie(&req);
|
let cookie_id = extract_session_cookie(&req);
|
||||||
|
|
||||||
let (session_id, session_data, is_new) =
|
let (session_id, session_data) =
|
||||||
match cookie_id.and_then(|id| state.store.get(&id).map(|data| (id.clone(), data))) {
|
match cookie_id.and_then(|id| state.store.get(&id).map(|data| (id.clone(), data))) {
|
||||||
Some((id, data)) => (id, data, false),
|
Some((id, data)) => (id, data),
|
||||||
None => {
|
None => state.store.create(),
|
||||||
let (id, data) = state.store.create();
|
|
||||||
(id, data, true)
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let handle = SessionHandle::new(session_id.clone(), session_data);
|
let handle = SessionHandle::new(session_id.clone(), session_data);
|
||||||
@@ -76,21 +94,32 @@ pub async fn session_layer(
|
|||||||
|
|
||||||
let mut resp = next.run(req).await;
|
let mut resp = next.run(req).await;
|
||||||
|
|
||||||
|
let rotated = handle.take_rotated_id();
|
||||||
|
let destroy_old = handle.take_destroy_old();
|
||||||
|
|
||||||
|
let effective_id = rotated.unwrap_or_else(|| handle.id.clone());
|
||||||
|
|
||||||
if handle.is_dirty() {
|
if handle.is_dirty() {
|
||||||
state.store.save(&handle.id, handle.snapshot());
|
state.store.save(&effective_id, handle.snapshot());
|
||||||
}
|
}
|
||||||
|
|
||||||
if is_new {
|
if let Some(old) = destroy_old {
|
||||||
let cookie = build_session_cookie(&session_id, state.secure);
|
state.store.destroy(&old);
|
||||||
if let Ok(value) = HeaderValue::from_str(&cookie.to_string()) {
|
}
|
||||||
resp.headers_mut().append(header::SET_COOKIE, value);
|
|
||||||
}
|
let cookie = build_session_cookie(&effective_id, state.secure, state.ttl);
|
||||||
|
if let Ok(value) = HeaderValue::from_str(&cookie.to_string()) {
|
||||||
|
resp.headers_mut().append(header::SET_COOKIE, value);
|
||||||
}
|
}
|
||||||
|
|
||||||
resp
|
resp
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn csrf_layer(req: Request, next: Next) -> Response {
|
pub async fn csrf_layer(
|
||||||
|
State(state): State<crate::state::AppState>,
|
||||||
|
req: Request,
|
||||||
|
next: Next,
|
||||||
|
) -> Response {
|
||||||
const CSRF_HEADER_ALIAS: &str = "x-csrftoken";
|
const CSRF_HEADER_ALIAS: &str = "x-csrftoken";
|
||||||
|
|
||||||
let method = req.method().clone();
|
let method = req.method().clone();
|
||||||
@@ -151,6 +180,8 @@ pub async fn csrf_layer(req: Request, next: Next) -> Response {
|
|||||||
extract_form_token(&bytes)
|
extract_form_token(&bytes)
|
||||||
} else if content_type.starts_with("multipart/form-data") {
|
} else if content_type.starts_with("multipart/form-data") {
|
||||||
extract_multipart_token(&content_type, &bytes)
|
extract_multipart_token(&content_type, &bytes)
|
||||||
|
} else if content_type.starts_with("application/json") {
|
||||||
|
extract_json_token(&bytes)
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
@@ -169,7 +200,32 @@ pub async fn csrf_layer(req: Request, next: Next) -> Response {
|
|||||||
header_present = header_token.is_some(),
|
header_present = header_token.is_some(),
|
||||||
"CSRF token mismatch"
|
"CSRF token mismatch"
|
||||||
);
|
);
|
||||||
(StatusCode::FORBIDDEN, "Invalid CSRF token").into_response()
|
|
||||||
|
let accept = parts
|
||||||
|
.headers
|
||||||
|
.get(header::ACCEPT)
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.unwrap_or("");
|
||||||
|
let is_form_submit = content_type.starts_with("application/x-www-form-urlencoded")
|
||||||
|
|| content_type.starts_with("multipart/form-data");
|
||||||
|
let wants_json =
|
||||||
|
accept.contains("application/json") || content_type.starts_with("application/json");
|
||||||
|
|
||||||
|
if is_form_submit && !wants_json {
|
||||||
|
let ctx = crate::handlers::ui::base_context(&handle, None);
|
||||||
|
let mut resp = crate::handlers::ui::render(&state, "csrf_error.html", &ctx);
|
||||||
|
*resp.status_mut() = StatusCode::FORBIDDEN;
|
||||||
|
return resp;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut resp = (
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
[(header::CONTENT_TYPE, "application/json")],
|
||||||
|
r#"{"error":"Invalid CSRF token. Send it via the X-CSRF-Token header or a csrf_token field in the form/JSON body."}"#,
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
*resp.status_mut() = StatusCode::FORBIDDEN;
|
||||||
|
resp
|
||||||
}
|
}
|
||||||
|
|
||||||
fn extract_multipart_token(content_type: &str, body: &[u8]) -> Option<String> {
|
fn extract_multipart_token(content_type: &str, body: &[u8]) -> Option<String> {
|
||||||
@@ -200,15 +256,25 @@ fn extract_session_cookie(req: &Request) -> Option<String> {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_session_cookie(id: &str, secure: bool) -> Cookie<'static> {
|
fn build_session_cookie(id: &str, secure: bool, ttl: Duration) -> Cookie<'static> {
|
||||||
let mut cookie = Cookie::new(SESSION_COOKIE_NAME, id.to_string());
|
let mut cookie = Cookie::new(SESSION_COOKIE_NAME, id.to_string());
|
||||||
cookie.set_http_only(true);
|
cookie.set_http_only(true);
|
||||||
cookie.set_same_site(SameSite::Lax);
|
cookie.set_same_site(SameSite::Strict);
|
||||||
cookie.set_secure(secure);
|
cookie.set_secure(secure);
|
||||||
cookie.set_path("/");
|
cookie.set_path("/");
|
||||||
|
let secs = i64::try_from(ttl.as_secs()).unwrap_or(i64::MAX);
|
||||||
|
cookie.set_max_age(CookieDuration::seconds(secs));
|
||||||
cookie
|
cookie
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn extract_json_token(body: &[u8]) -> Option<String> {
|
||||||
|
let value: serde_json::Value = serde_json::from_slice(body).ok()?;
|
||||||
|
value
|
||||||
|
.get(CSRF_FIELD_NAME)
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
fn extract_form_token(body: &[u8]) -> Option<String> {
|
fn extract_form_token(body: &[u8]) -> Option<String> {
|
||||||
let text = std::str::from_utf8(body).ok()?;
|
let text = std::str::from_utf8(body).ok()?;
|
||||||
let prefix = format!("{}=", CSRF_FIELD_NAME);
|
let prefix = format!("{}=", CSRF_FIELD_NAME);
|
||||||
|
|||||||
107
crates/myfsio-server/src/middleware/sha_body.rs
Normal file
107
crates/myfsio-server/src/middleware/sha_body.rs
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
use axum::body::Body;
|
||||||
|
use bytes::Bytes;
|
||||||
|
use http_body::{Body as HttpBody, Frame};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use std::error::Error;
|
||||||
|
use std::fmt;
|
||||||
|
use std::pin::Pin;
|
||||||
|
use std::task::{Context, Poll};
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct Sha256MismatchError {
|
||||||
|
expected: String,
|
||||||
|
computed: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Sha256MismatchError {
|
||||||
|
fn message(&self) -> String {
|
||||||
|
format!(
|
||||||
|
"The x-amz-content-sha256 you specified did not match what we received (expected {}, computed {})",
|
||||||
|
self.expected, self.computed
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Display for Sha256MismatchError {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
write!(
|
||||||
|
f,
|
||||||
|
"XAmzContentSHA256Mismatch: expected {}, computed {}",
|
||||||
|
self.expected, self.computed
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Error for Sha256MismatchError {}
|
||||||
|
|
||||||
|
pub struct Sha256VerifyBody {
|
||||||
|
inner: Body,
|
||||||
|
expected: String,
|
||||||
|
hasher: Option<Sha256>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Sha256VerifyBody {
|
||||||
|
pub fn new(inner: Body, expected_hex: String) -> Self {
|
||||||
|
Self {
|
||||||
|
inner,
|
||||||
|
expected: expected_hex.to_ascii_lowercase(),
|
||||||
|
hasher: Some(Sha256::new()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HttpBody for Sha256VerifyBody {
|
||||||
|
type Data = Bytes;
|
||||||
|
type Error = Box<dyn std::error::Error + Send + Sync>;
|
||||||
|
|
||||||
|
fn poll_frame(
|
||||||
|
mut self: Pin<&mut Self>,
|
||||||
|
cx: &mut Context<'_>,
|
||||||
|
) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
|
||||||
|
let this = self.as_mut().get_mut();
|
||||||
|
match Pin::new(&mut this.inner).poll_frame(cx) {
|
||||||
|
Poll::Pending => Poll::Pending,
|
||||||
|
Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(Box::new(e)))),
|
||||||
|
Poll::Ready(Some(Ok(frame))) => {
|
||||||
|
if let Some(data) = frame.data_ref() {
|
||||||
|
if let Some(h) = this.hasher.as_mut() {
|
||||||
|
h.update(data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Poll::Ready(Some(Ok(frame)))
|
||||||
|
}
|
||||||
|
Poll::Ready(None) => {
|
||||||
|
if let Some(hasher) = this.hasher.take() {
|
||||||
|
let computed = hex::encode(hasher.finalize());
|
||||||
|
if computed != this.expected {
|
||||||
|
return Poll::Ready(Some(Err(Box::new(Sha256MismatchError {
|
||||||
|
expected: this.expected.clone(),
|
||||||
|
computed,
|
||||||
|
}))));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Poll::Ready(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_end_stream(&self) -> bool {
|
||||||
|
self.inner.is_end_stream()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn size_hint(&self) -> http_body::SizeHint {
|
||||||
|
self.inner.size_hint()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_hex_sha256(s: &str) -> bool {
|
||||||
|
s.len() == 64 && s.bytes().all(|b| b.is_ascii_hexdigit())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn sha256_mismatch_message(err: &(dyn Error + 'static)) -> Option<String> {
|
||||||
|
if let Some(mismatch) = err.downcast_ref::<Sha256MismatchError>() {
|
||||||
|
return Some(mismatch.message());
|
||||||
|
}
|
||||||
|
|
||||||
|
err.source().and_then(sha256_mismatch_message)
|
||||||
|
}
|
||||||
@@ -9,6 +9,7 @@ pub struct GcConfig {
|
|||||||
pub temp_file_max_age_hours: f64,
|
pub temp_file_max_age_hours: f64,
|
||||||
pub multipart_max_age_days: u64,
|
pub multipart_max_age_days: u64,
|
||||||
pub lock_file_max_age_hours: f64,
|
pub lock_file_max_age_hours: f64,
|
||||||
|
pub quarantine_max_age_days: u64,
|
||||||
pub dry_run: bool,
|
pub dry_run: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -19,6 +20,7 @@ impl Default for GcConfig {
|
|||||||
temp_file_max_age_hours: 24.0,
|
temp_file_max_age_hours: 24.0,
|
||||||
multipart_max_age_days: 7,
|
multipart_max_age_days: 7,
|
||||||
lock_file_max_age_hours: 1.0,
|
lock_file_max_age_hours: 1.0,
|
||||||
|
quarantine_max_age_days: 7,
|
||||||
dry_run: false,
|
dry_run: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -106,6 +108,7 @@ impl GcService {
|
|||||||
"temp_file_max_age_hours": self.config.temp_file_max_age_hours,
|
"temp_file_max_age_hours": self.config.temp_file_max_age_hours,
|
||||||
"multipart_max_age_days": self.config.multipart_max_age_days,
|
"multipart_max_age_days": self.config.multipart_max_age_days,
|
||||||
"lock_file_max_age_hours": self.config.lock_file_max_age_hours,
|
"lock_file_max_age_hours": self.config.lock_file_max_age_hours,
|
||||||
|
"quarantine_max_age_days": self.config.quarantine_max_age_days,
|
||||||
"dry_run": self.config.dry_run,
|
"dry_run": self.config.dry_run,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -164,6 +167,8 @@ impl GcService {
|
|||||||
let mut multipart_uploads_deleted = 0u64;
|
let mut multipart_uploads_deleted = 0u64;
|
||||||
let mut lock_files_deleted = 0u64;
|
let mut lock_files_deleted = 0u64;
|
||||||
let mut empty_dirs_removed = 0u64;
|
let mut empty_dirs_removed = 0u64;
|
||||||
|
let mut quarantine_entries_deleted = 0u64;
|
||||||
|
let mut quarantine_bytes_freed = 0u64;
|
||||||
let mut errors: Vec<String> = Vec::new();
|
let mut errors: Vec<String> = Vec::new();
|
||||||
|
|
||||||
let now = std::time::SystemTime::now();
|
let now = std::time::SystemTime::now();
|
||||||
@@ -173,6 +178,8 @@ impl GcService {
|
|||||||
std::time::Duration::from_secs(self.config.multipart_max_age_days * 86400);
|
std::time::Duration::from_secs(self.config.multipart_max_age_days * 86400);
|
||||||
let lock_max_age =
|
let lock_max_age =
|
||||||
std::time::Duration::from_secs_f64(self.config.lock_file_max_age_hours * 3600.0);
|
std::time::Duration::from_secs_f64(self.config.lock_file_max_age_hours * 3600.0);
|
||||||
|
let quarantine_max_age =
|
||||||
|
std::time::Duration::from_secs(self.config.quarantine_max_age_days * 86400);
|
||||||
|
|
||||||
let tmp_dir = self.storage_root.join(".myfsio.sys").join("tmp");
|
let tmp_dir = self.storage_root.join(".myfsio.sys").join("tmp");
|
||||||
if tmp_dir.exists() {
|
if tmp_dir.exists() {
|
||||||
@@ -256,6 +263,55 @@ impl GcService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let quarantine_dir = self.storage_root.join(".myfsio.sys").join("quarantine");
|
||||||
|
if quarantine_dir.exists() {
|
||||||
|
if let Ok(bucket_dirs) = std::fs::read_dir(&quarantine_dir) {
|
||||||
|
for bucket_entry in bucket_dirs.flatten() {
|
||||||
|
if !bucket_entry.path().is_dir() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Ok(ts_dirs) = std::fs::read_dir(bucket_entry.path()) {
|
||||||
|
for ts_entry in ts_dirs.flatten() {
|
||||||
|
let ts_path = ts_entry.path();
|
||||||
|
if !ts_path.is_dir() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let modified = ts_entry.metadata().ok().and_then(|m| m.modified().ok());
|
||||||
|
let Some(modified) = modified else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Ok(age) = now.duration_since(modified) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if age <= quarantine_max_age {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let bytes = dir_total_bytes(&ts_path);
|
||||||
|
if !dry_run {
|
||||||
|
if let Err(e) = std::fs::remove_dir_all(&ts_path) {
|
||||||
|
errors.push(format!(
|
||||||
|
"Failed to remove quarantine {}: {}",
|
||||||
|
ts_path.display(),
|
||||||
|
e
|
||||||
|
));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
quarantine_entries_deleted += 1;
|
||||||
|
quarantine_bytes_freed += bytes;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !dry_run {
|
||||||
|
if let Ok(mut remaining) = std::fs::read_dir(bucket_entry.path()) {
|
||||||
|
if remaining.next().is_none() {
|
||||||
|
let _ = std::fs::remove_dir(bucket_entry.path());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if !dry_run {
|
if !dry_run {
|
||||||
for dir in [&tmp_dir, &multipart_dir] {
|
for dir in [&tmp_dir, &multipart_dir] {
|
||||||
if dir.exists() {
|
if dir.exists() {
|
||||||
@@ -281,6 +337,8 @@ impl GcService {
|
|||||||
"multipart_uploads_deleted": multipart_uploads_deleted,
|
"multipart_uploads_deleted": multipart_uploads_deleted,
|
||||||
"lock_files_deleted": lock_files_deleted,
|
"lock_files_deleted": lock_files_deleted,
|
||||||
"empty_dirs_removed": empty_dirs_removed,
|
"empty_dirs_removed": empty_dirs_removed,
|
||||||
|
"quarantine_entries_deleted": quarantine_entries_deleted,
|
||||||
|
"quarantine_bytes_freed": quarantine_bytes_freed,
|
||||||
"errors": errors,
|
"errors": errors,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -313,3 +371,22 @@ impl GcService {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn dir_total_bytes(path: &std::path::Path) -> u64 {
|
||||||
|
let mut total: u64 = 0;
|
||||||
|
let mut stack: Vec<PathBuf> = vec![path.to_path_buf()];
|
||||||
|
while let Some(dir) = stack.pop() {
|
||||||
|
let Ok(entries) = std::fs::read_dir(&dir) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let Ok(ft) = entry.file_type() else { continue };
|
||||||
|
if ft.is_dir() {
|
||||||
|
stack.push(entry.path());
|
||||||
|
} else if ft.is_file() {
|
||||||
|
total = total.saturating_add(entry.metadata().map(|m| m.len()).unwrap_or(0));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
total
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,22 +1,31 @@
|
|||||||
use myfsio_common::constants::{
|
use myfsio_common::constants::{
|
||||||
BUCKET_META_DIR, BUCKET_VERSIONS_DIR, INDEX_FILE, SYSTEM_BUCKETS_DIR, SYSTEM_ROOT,
|
BUCKET_META_DIR, BUCKET_VERSIONS_DIR, INDEX_FILE, SYSTEM_BUCKETS_DIR, SYSTEM_ROOT,
|
||||||
};
|
};
|
||||||
use myfsio_storage::fs_backend::FsStorageBackend;
|
use myfsio_storage::fs_backend::{
|
||||||
|
is_multipart_etag, metadata_is_corrupted, FsStorageBackend, META_KEY_CORRUPTED,
|
||||||
|
META_KEY_CORRUPTED_AT, META_KEY_CORRUPTION_DETAIL, META_KEY_QUARANTINE_PATH,
|
||||||
|
};
|
||||||
|
use myfsio_storage::traits::StorageEngine;
|
||||||
use serde_json::{json, Map, Value};
|
use serde_json::{json, Map, Value};
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
use tokio::sync::RwLock;
|
use tokio::sync::{RwLock, Semaphore};
|
||||||
|
|
||||||
|
use crate::services::peer_fetch::{HealOutcome, PeerFetcher};
|
||||||
|
|
||||||
const MAX_ISSUES: usize = 500;
|
const MAX_ISSUES: usize = 500;
|
||||||
const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"];
|
const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"];
|
||||||
|
const QUARANTINE_DIR: &str = "quarantine";
|
||||||
|
|
||||||
pub struct IntegrityConfig {
|
pub struct IntegrityConfig {
|
||||||
pub interval_hours: f64,
|
pub interval_hours: f64,
|
||||||
pub batch_size: usize,
|
pub batch_size: usize,
|
||||||
pub auto_heal: bool,
|
pub auto_heal: bool,
|
||||||
pub dry_run: bool,
|
pub dry_run: bool,
|
||||||
|
pub heal_concurrency: usize,
|
||||||
|
pub quarantine_retention_days: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for IntegrityConfig {
|
impl Default for IntegrityConfig {
|
||||||
@@ -26,21 +35,50 @@ impl Default for IntegrityConfig {
|
|||||||
batch_size: 10_000,
|
batch_size: 10_000,
|
||||||
auto_heal: false,
|
auto_heal: false,
|
||||||
dry_run: false,
|
dry_run: false,
|
||||||
|
heal_concurrency: 4,
|
||||||
|
quarantine_retention_days: 7,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct IntegrityService {
|
pub struct IntegrityService {
|
||||||
#[allow(dead_code)]
|
|
||||||
storage: Arc<FsStorageBackend>,
|
storage: Arc<FsStorageBackend>,
|
||||||
storage_root: PathBuf,
|
storage_root: PathBuf,
|
||||||
config: IntegrityConfig,
|
config: IntegrityConfig,
|
||||||
|
peer_fetcher: Option<Arc<PeerFetcher>>,
|
||||||
running: Arc<RwLock<bool>>,
|
running: Arc<RwLock<bool>>,
|
||||||
started_at: Arc<RwLock<Option<Instant>>>,
|
started_at: Arc<RwLock<Option<Instant>>>,
|
||||||
history: Arc<RwLock<Vec<Value>>>,
|
history: Arc<RwLock<Vec<Value>>>,
|
||||||
history_path: PathBuf,
|
history_path: PathBuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Default, Clone)]
|
||||||
|
struct HealStats {
|
||||||
|
found: u64,
|
||||||
|
healed: u64,
|
||||||
|
poisoned: u64,
|
||||||
|
peer_mismatch: u64,
|
||||||
|
peer_unavailable: u64,
|
||||||
|
verify_failed: u64,
|
||||||
|
failed: u64,
|
||||||
|
skipped: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HealStats {
|
||||||
|
fn to_value(&self) -> Value {
|
||||||
|
json!({
|
||||||
|
"found": self.found,
|
||||||
|
"healed": self.healed,
|
||||||
|
"poisoned": self.poisoned,
|
||||||
|
"peer_mismatch": self.peer_mismatch,
|
||||||
|
"peer_unavailable": self.peer_unavailable,
|
||||||
|
"verify_failed": self.verify_failed,
|
||||||
|
"failed": self.failed,
|
||||||
|
"skipped": self.skipped,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct ScanState {
|
struct ScanState {
|
||||||
objects_scanned: u64,
|
objects_scanned: u64,
|
||||||
@@ -69,22 +107,6 @@ impl ScanState {
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn into_json(self, elapsed: f64) -> Value {
|
|
||||||
json!({
|
|
||||||
"objects_scanned": self.objects_scanned,
|
|
||||||
"buckets_scanned": self.buckets_scanned,
|
|
||||||
"corrupted_objects": self.corrupted_objects,
|
|
||||||
"orphaned_objects": self.orphaned_objects,
|
|
||||||
"phantom_metadata": self.phantom_metadata,
|
|
||||||
"stale_versions": self.stale_versions,
|
|
||||||
"etag_cache_inconsistencies": self.etag_cache_inconsistencies,
|
|
||||||
"issues_healed": 0,
|
|
||||||
"issues": self.issues,
|
|
||||||
"errors": self.errors,
|
|
||||||
"execution_time_seconds": elapsed,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl IntegrityService {
|
impl IntegrityService {
|
||||||
@@ -92,6 +114,7 @@ impl IntegrityService {
|
|||||||
storage: Arc<FsStorageBackend>,
|
storage: Arc<FsStorageBackend>,
|
||||||
storage_root: &Path,
|
storage_root: &Path,
|
||||||
config: IntegrityConfig,
|
config: IntegrityConfig,
|
||||||
|
peer_fetcher: Option<Arc<PeerFetcher>>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let history_path = storage_root
|
let history_path = storage_root
|
||||||
.join(SYSTEM_ROOT)
|
.join(SYSTEM_ROOT)
|
||||||
@@ -112,6 +135,7 @@ impl IntegrityService {
|
|||||||
storage,
|
storage,
|
||||||
storage_root: storage_root.to_path_buf(),
|
storage_root: storage_root.to_path_buf(),
|
||||||
config,
|
config,
|
||||||
|
peer_fetcher,
|
||||||
running: Arc::new(RwLock::new(false)),
|
running: Arc::new(RwLock::new(false)),
|
||||||
started_at: Arc::new(RwLock::new(None)),
|
started_at: Arc::new(RwLock::new(None)),
|
||||||
history: Arc::new(RwLock::new(history)),
|
history: Arc::new(RwLock::new(history)),
|
||||||
@@ -136,6 +160,8 @@ impl IntegrityService {
|
|||||||
"batch_size": self.config.batch_size,
|
"batch_size": self.config.batch_size,
|
||||||
"auto_heal": self.config.auto_heal,
|
"auto_heal": self.config.auto_heal,
|
||||||
"dry_run": self.config.dry_run,
|
"dry_run": self.config.dry_run,
|
||||||
|
"heal_concurrency": self.config.heal_concurrency,
|
||||||
|
"peer_heal_available": self.peer_fetcher.is_some(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -159,7 +185,7 @@ impl IntegrityService {
|
|||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
let storage_root = self.storage_root.clone();
|
let storage_root = self.storage_root.clone();
|
||||||
let batch_size = self.config.batch_size;
|
let batch_size = self.config.batch_size;
|
||||||
let result =
|
let scan_state =
|
||||||
tokio::task::spawn_blocking(move || scan_all_buckets(&storage_root, batch_size))
|
tokio::task::spawn_blocking(move || scan_all_buckets(&storage_root, batch_size))
|
||||||
.await
|
.await
|
||||||
.unwrap_or_else(|e| {
|
.unwrap_or_else(|e| {
|
||||||
@@ -167,12 +193,19 @@ impl IntegrityService {
|
|||||||
st.errors.push(format!("scan task failed: {}", e));
|
st.errors.push(format!("scan task failed: {}", e));
|
||||||
st
|
st
|
||||||
});
|
});
|
||||||
|
|
||||||
|
let heal_stats = if auto_heal && !dry_run {
|
||||||
|
self.run_heal_phase(&scan_state).await
|
||||||
|
} else {
|
||||||
|
BTreeMap::new()
|
||||||
|
};
|
||||||
|
|
||||||
let elapsed = start.elapsed().as_secs_f64();
|
let elapsed = start.elapsed().as_secs_f64();
|
||||||
|
|
||||||
*self.running.write().await = false;
|
*self.running.write().await = false;
|
||||||
*self.started_at.write().await = None;
|
*self.started_at.write().await = None;
|
||||||
|
|
||||||
let result_json = result.into_json(elapsed);
|
let result_json = build_result_json(scan_state, heal_stats, elapsed);
|
||||||
|
|
||||||
let record = json!({
|
let record = json!({
|
||||||
"timestamp": chrono::Utc::now().timestamp_millis() as f64 / 1000.0,
|
"timestamp": chrono::Utc::now().timestamp_millis() as f64 / 1000.0,
|
||||||
@@ -194,6 +227,77 @@ impl IntegrityService {
|
|||||||
Ok(result_json)
|
Ok(result_json)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn run_heal_phase(&self, scan: &ScanState) -> BTreeMap<String, HealStats> {
|
||||||
|
let mut stats: BTreeMap<String, HealStats> = BTreeMap::new();
|
||||||
|
let issues: Vec<Value> = scan.issues.clone();
|
||||||
|
let semaphore = Arc::new(Semaphore::new(self.config.heal_concurrency.max(1)));
|
||||||
|
let mut tasks: Vec<tokio::task::JoinHandle<HealReport>> = Vec::new();
|
||||||
|
|
||||||
|
for issue in issues {
|
||||||
|
let issue_type = issue
|
||||||
|
.get("issue_type")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
let bucket = issue
|
||||||
|
.get("bucket")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
let key = issue
|
||||||
|
.get("key")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
let detail = issue
|
||||||
|
.get("detail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
stats.entry(issue_type.clone()).or_default().found += 1;
|
||||||
|
|
||||||
|
let permit = match semaphore.clone().acquire_owned().await {
|
||||||
|
Ok(p) => p,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
let storage = self.storage.clone();
|
||||||
|
let storage_root = self.storage_root.clone();
|
||||||
|
let peer_fetcher = self.peer_fetcher.clone();
|
||||||
|
|
||||||
|
tasks.push(tokio::spawn(async move {
|
||||||
|
let _permit = permit;
|
||||||
|
heal_issue(
|
||||||
|
&storage,
|
||||||
|
&storage_root,
|
||||||
|
peer_fetcher.as_deref(),
|
||||||
|
&issue_type,
|
||||||
|
&bucket,
|
||||||
|
&key,
|
||||||
|
&detail,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
for task in tasks {
|
||||||
|
if let Ok(report) = task.await {
|
||||||
|
let entry = stats.entry(report.issue_type).or_default();
|
||||||
|
match report.status {
|
||||||
|
HealStatus::Healed => entry.healed += 1,
|
||||||
|
HealStatus::Poisoned => entry.poisoned += 1,
|
||||||
|
HealStatus::PeerMismatch => entry.peer_mismatch += 1,
|
||||||
|
HealStatus::PeerUnavailable => entry.peer_unavailable += 1,
|
||||||
|
HealStatus::VerifyFailed => entry.verify_failed += 1,
|
||||||
|
HealStatus::Failed => entry.failed += 1,
|
||||||
|
HealStatus::Skipped => entry.skipped += 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stats
|
||||||
|
}
|
||||||
|
|
||||||
async fn save_history(&self) {
|
async fn save_history(&self) {
|
||||||
let history = self.history.read().await;
|
let history = self.history.read().await;
|
||||||
let data = json!({ "executions": *history });
|
let data = json!({ "executions": *history });
|
||||||
@@ -208,13 +312,15 @@ impl IntegrityService {
|
|||||||
|
|
||||||
pub fn start_background(self: Arc<Self>) -> tokio::task::JoinHandle<()> {
|
pub fn start_background(self: Arc<Self>) -> tokio::task::JoinHandle<()> {
|
||||||
let interval = std::time::Duration::from_secs_f64(self.config.interval_hours * 3600.0);
|
let interval = std::time::Duration::from_secs_f64(self.config.interval_hours * 3600.0);
|
||||||
|
let auto_heal = self.config.auto_heal;
|
||||||
|
let dry_run = self.config.dry_run;
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let mut timer = tokio::time::interval(interval);
|
let mut timer = tokio::time::interval(interval);
|
||||||
timer.tick().await;
|
timer.tick().await;
|
||||||
loop {
|
loop {
|
||||||
timer.tick().await;
|
timer.tick().await;
|
||||||
tracing::info!("Integrity check starting");
|
tracing::info!("Integrity check starting");
|
||||||
match self.run_now(false, false).await {
|
match self.run_now(dry_run, auto_heal).await {
|
||||||
Ok(result) => tracing::info!("Integrity check complete: {:?}", result),
|
Ok(result) => tracing::info!("Integrity check complete: {:?}", result),
|
||||||
Err(e) => tracing::warn!("Integrity check failed: {}", e),
|
Err(e) => tracing::warn!("Integrity check failed: {}", e),
|
||||||
}
|
}
|
||||||
@@ -223,6 +329,395 @@ impl IntegrityService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
enum HealStatus {
|
||||||
|
Healed,
|
||||||
|
Poisoned,
|
||||||
|
PeerMismatch,
|
||||||
|
PeerUnavailable,
|
||||||
|
VerifyFailed,
|
||||||
|
Failed,
|
||||||
|
Skipped,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct HealReport {
|
||||||
|
issue_type: String,
|
||||||
|
status: HealStatus,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn heal_issue(
|
||||||
|
storage: &FsStorageBackend,
|
||||||
|
storage_root: &Path,
|
||||||
|
peer_fetcher: Option<&PeerFetcher>,
|
||||||
|
issue_type: &str,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
detail: &str,
|
||||||
|
) -> HealReport {
|
||||||
|
let status = match issue_type {
|
||||||
|
"corrupted_object" => {
|
||||||
|
heal_corrupted(storage, storage_root, peer_fetcher, bucket, key, detail).await
|
||||||
|
}
|
||||||
|
"stale_version" => heal_stale_version(storage_root, bucket, key).await,
|
||||||
|
"etag_cache_inconsistency" => heal_etag_cache(storage_root, bucket, key, detail).await,
|
||||||
|
"phantom_metadata" => heal_phantom_metadata(storage, bucket, key).await,
|
||||||
|
_ => HealStatus::Skipped,
|
||||||
|
};
|
||||||
|
HealReport {
|
||||||
|
issue_type: issue_type.to_string(),
|
||||||
|
status,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn heal_corrupted(
|
||||||
|
storage: &FsStorageBackend,
|
||||||
|
storage_root: &Path,
|
||||||
|
peer_fetcher: Option<&PeerFetcher>,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
detail: &str,
|
||||||
|
) -> HealStatus {
|
||||||
|
let stored_etag = parse_stored_etag(detail);
|
||||||
|
let actual_etag = parse_actual_etag(detail);
|
||||||
|
|
||||||
|
let live_path = storage_root.join(bucket).join(key);
|
||||||
|
let quarantine_rel = quarantine_relative_path(bucket, key);
|
||||||
|
let quarantine_full = storage_root.join(&quarantine_rel);
|
||||||
|
|
||||||
|
if let Some(parent) = quarantine_full.parent() {
|
||||||
|
if let Err(e) = std::fs::create_dir_all(parent) {
|
||||||
|
tracing::error!("Heal {}/{}: mkdir quarantine failed: {}", bucket, key, e);
|
||||||
|
return HealStatus::Failed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
let _guard = storage.lock_object_write(bucket, key);
|
||||||
|
if live_path.exists() {
|
||||||
|
if let Err(e) = std::fs::rename(&live_path, &quarantine_full) {
|
||||||
|
tracing::error!("Heal {}/{}: quarantine rename failed: {}", bucket, key, e);
|
||||||
|
return HealStatus::Failed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let quarantine_rel_str = quarantine_rel.to_string_lossy().replace('\\', "/");
|
||||||
|
|
||||||
|
if !stored_etag.is_empty() {
|
||||||
|
if let Some(fetcher) = peer_fetcher {
|
||||||
|
let nonce = uuid::Uuid::new_v4().simple().to_string();
|
||||||
|
let temp_path = live_path.with_file_name(format!(
|
||||||
|
"{}.healing.{}",
|
||||||
|
live_path
|
||||||
|
.file_name()
|
||||||
|
.map(|n| n.to_string_lossy().into_owned())
|
||||||
|
.unwrap_or_else(|| "healing".to_string()),
|
||||||
|
nonce
|
||||||
|
));
|
||||||
|
match fetcher
|
||||||
|
.fetch_for_heal(bucket, key, &stored_etag, &temp_path)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
HealOutcome::Healed { peer_etag, bytes } => {
|
||||||
|
let swap_result = {
|
||||||
|
let _guard = storage.lock_object_write(bucket, key);
|
||||||
|
if live_path.exists() {
|
||||||
|
let _ = std::fs::remove_file(&temp_path);
|
||||||
|
tracing::info!(
|
||||||
|
"Heal {}/{}: concurrent PUT raced; preserving fresh write",
|
||||||
|
bucket,
|
||||||
|
key
|
||||||
|
);
|
||||||
|
return HealStatus::Skipped;
|
||||||
|
}
|
||||||
|
atomic_swap(&temp_path, &live_path)
|
||||||
|
};
|
||||||
|
if let Err(e) = swap_result {
|
||||||
|
tracing::error!(
|
||||||
|
"Heal {}/{}: atomic swap failed: {} (restoring from quarantine)",
|
||||||
|
bucket,
|
||||||
|
key,
|
||||||
|
e
|
||||||
|
);
|
||||||
|
let _guard = storage.lock_object_write(bucket, key);
|
||||||
|
if !live_path.exists() {
|
||||||
|
let _ = std::fs::rename(&quarantine_full, &live_path);
|
||||||
|
}
|
||||||
|
let _ = std::fs::remove_file(&temp_path);
|
||||||
|
return HealStatus::Failed;
|
||||||
|
}
|
||||||
|
let _ = clear_poison_metadata(storage, bucket, key).await;
|
||||||
|
tracing::info!(
|
||||||
|
"Healed {}/{} from peer (etag={}, bytes={})",
|
||||||
|
bucket,
|
||||||
|
key,
|
||||||
|
peer_etag,
|
||||||
|
bytes
|
||||||
|
);
|
||||||
|
return HealStatus::Healed;
|
||||||
|
}
|
||||||
|
HealOutcome::PeerMismatch { stored, peer } => {
|
||||||
|
let msg = format!("peer etag {} != stored {}", peer, stored);
|
||||||
|
let _ = poison_metadata(storage, bucket, key, &msg, &quarantine_rel_str).await;
|
||||||
|
tracing::warn!("Heal {}/{}: peer mismatch ({}), poisoned", bucket, key, msg);
|
||||||
|
return HealStatus::PeerMismatch;
|
||||||
|
}
|
||||||
|
HealOutcome::PeerUnavailable { error } => {
|
||||||
|
tracing::warn!(
|
||||||
|
"Heal {}/{}: peer unavailable ({}), poisoning",
|
||||||
|
bucket,
|
||||||
|
key,
|
||||||
|
error
|
||||||
|
);
|
||||||
|
let msg = format!(
|
||||||
|
"etag mismatch (stored={}, actual={}) — peer unavailable: {}",
|
||||||
|
stored_etag, actual_etag, error
|
||||||
|
);
|
||||||
|
let _ = poison_metadata(storage, bucket, key, &msg, &quarantine_rel_str).await;
|
||||||
|
return HealStatus::PeerUnavailable;
|
||||||
|
}
|
||||||
|
HealOutcome::VerifyFailed { expected, actual } => {
|
||||||
|
let msg = format!(
|
||||||
|
"peer download verify failed: expected={} actual={}",
|
||||||
|
expected, actual
|
||||||
|
);
|
||||||
|
let _ = poison_metadata(storage, bucket, key, &msg, &quarantine_rel_str).await;
|
||||||
|
tracing::warn!("Heal {}/{}: {}", bucket, key, msg);
|
||||||
|
return HealStatus::VerifyFailed;
|
||||||
|
}
|
||||||
|
HealOutcome::NotConfigured => {
|
||||||
|
let msg = format!(
|
||||||
|
"etag mismatch (stored={}, actual={}); no peer configured",
|
||||||
|
stored_etag, actual_etag
|
||||||
|
);
|
||||||
|
let _ = poison_metadata(storage, bucket, key, &msg, &quarantine_rel_str).await;
|
||||||
|
return HealStatus::Poisoned;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let msg = format!(
|
||||||
|
"etag mismatch (stored={}, actual={}); no peer fetcher",
|
||||||
|
stored_etag, actual_etag
|
||||||
|
);
|
||||||
|
let _ = poison_metadata(storage, bucket, key, &msg, &quarantine_rel_str).await;
|
||||||
|
HealStatus::Poisoned
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn heal_stale_version(storage_root: &Path, bucket: &str, key: &str) -> HealStatus {
|
||||||
|
let versions_root = storage_root
|
||||||
|
.join(SYSTEM_ROOT)
|
||||||
|
.join(SYSTEM_BUCKETS_DIR)
|
||||||
|
.join(bucket)
|
||||||
|
.join(BUCKET_VERSIONS_DIR);
|
||||||
|
let src = versions_root.join(key);
|
||||||
|
if !src.exists() {
|
||||||
|
return HealStatus::Skipped;
|
||||||
|
}
|
||||||
|
let ts = chrono::Utc::now().format("%Y%m%dT%H%M%S").to_string();
|
||||||
|
let dst = storage_root
|
||||||
|
.join(SYSTEM_ROOT)
|
||||||
|
.join(QUARANTINE_DIR)
|
||||||
|
.join(bucket)
|
||||||
|
.join(&ts)
|
||||||
|
.join("versions")
|
||||||
|
.join(key);
|
||||||
|
if let Some(parent) = dst.parent() {
|
||||||
|
if let Err(e) = std::fs::create_dir_all(parent) {
|
||||||
|
tracing::error!(
|
||||||
|
"Stale-version quarantine mkdir failed {}/{}: {}",
|
||||||
|
bucket,
|
||||||
|
key,
|
||||||
|
e
|
||||||
|
);
|
||||||
|
return HealStatus::Failed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Err(e) = std::fs::rename(&src, &dst) {
|
||||||
|
tracing::error!(
|
||||||
|
"Stale-version quarantine rename failed {}/{}: {}",
|
||||||
|
bucket,
|
||||||
|
key,
|
||||||
|
e
|
||||||
|
);
|
||||||
|
return HealStatus::Failed;
|
||||||
|
}
|
||||||
|
tracing::info!("Quarantined stale version {}/{}", bucket, key);
|
||||||
|
HealStatus::Healed
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn heal_etag_cache(
|
||||||
|
storage_root: &Path,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
_detail: &str,
|
||||||
|
) -> HealStatus {
|
||||||
|
let etag_index_path = storage_root
|
||||||
|
.join(SYSTEM_ROOT)
|
||||||
|
.join(SYSTEM_BUCKETS_DIR)
|
||||||
|
.join(bucket)
|
||||||
|
.join("etag_index.json");
|
||||||
|
if !etag_index_path.exists() {
|
||||||
|
return HealStatus::Skipped;
|
||||||
|
}
|
||||||
|
|
||||||
|
let meta_root = storage_root
|
||||||
|
.join(SYSTEM_ROOT)
|
||||||
|
.join(SYSTEM_BUCKETS_DIR)
|
||||||
|
.join(bucket)
|
||||||
|
.join(BUCKET_META_DIR);
|
||||||
|
let entries = collect_index_entries(&meta_root);
|
||||||
|
let canonical = entries.get(key).and_then(|info| stored_etag(&info.entry));
|
||||||
|
|
||||||
|
let mut cache: HashMap<String, Value> = match std::fs::read_to_string(&etag_index_path)
|
||||||
|
.ok()
|
||||||
|
.and_then(|s| serde_json::from_str(&s).ok())
|
||||||
|
{
|
||||||
|
Some(Value::Object(m)) => m.into_iter().collect(),
|
||||||
|
_ => return HealStatus::Failed,
|
||||||
|
};
|
||||||
|
|
||||||
|
match canonical {
|
||||||
|
Some(etag) => {
|
||||||
|
cache.insert(key.to_string(), Value::String(etag));
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
cache.remove(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let json_obj: serde_json::Map<String, Value> = cache.into_iter().collect();
|
||||||
|
match std::fs::write(
|
||||||
|
&etag_index_path,
|
||||||
|
serde_json::to_string_pretty(&Value::Object(json_obj)).unwrap_or_default(),
|
||||||
|
) {
|
||||||
|
Ok(_) => HealStatus::Healed,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!("etag-cache rewrite failed {}/{}: {}", bucket, key, e);
|
||||||
|
HealStatus::Failed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn heal_phantom_metadata(storage: &FsStorageBackend, bucket: &str, key: &str) -> HealStatus {
|
||||||
|
match storage.delete_object_metadata_entry(bucket, key).await {
|
||||||
|
Ok(_) => {
|
||||||
|
tracing::info!("Dropped phantom metadata for {}/{}", bucket, key);
|
||||||
|
HealStatus::Healed
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!("Failed to drop phantom metadata {}/{}: {}", bucket, key, e);
|
||||||
|
HealStatus::Failed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn poison_metadata(
|
||||||
|
storage: &FsStorageBackend,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
detail: &str,
|
||||||
|
quarantine_rel: &str,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let mut meta = storage
|
||||||
|
.get_object_metadata(bucket, key)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
meta.insert(META_KEY_CORRUPTED.to_string(), "true".to_string());
|
||||||
|
meta.insert(
|
||||||
|
META_KEY_CORRUPTED_AT.to_string(),
|
||||||
|
chrono::Utc::now().to_rfc3339(),
|
||||||
|
);
|
||||||
|
meta.insert(META_KEY_CORRUPTION_DETAIL.to_string(), detail.to_string());
|
||||||
|
meta.insert(
|
||||||
|
META_KEY_QUARANTINE_PATH.to_string(),
|
||||||
|
quarantine_rel.to_string(),
|
||||||
|
);
|
||||||
|
storage
|
||||||
|
.put_object_metadata(bucket, key, &meta)
|
||||||
|
.await
|
||||||
|
.map_err(|e| e.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn clear_poison_metadata(
|
||||||
|
storage: &FsStorageBackend,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let mut meta = storage
|
||||||
|
.get_object_metadata(bucket, key)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
meta.remove(META_KEY_CORRUPTED);
|
||||||
|
meta.remove(META_KEY_CORRUPTED_AT);
|
||||||
|
meta.remove(META_KEY_CORRUPTION_DETAIL);
|
||||||
|
meta.remove(META_KEY_QUARANTINE_PATH);
|
||||||
|
storage
|
||||||
|
.put_object_metadata(bucket, key, &meta)
|
||||||
|
.await
|
||||||
|
.map_err(|e| e.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn quarantine_relative_path(bucket: &str, key: &str) -> PathBuf {
|
||||||
|
let ts = chrono::Utc::now().format("%Y%m%dT%H%M%S").to_string();
|
||||||
|
PathBuf::from(SYSTEM_ROOT)
|
||||||
|
.join(QUARANTINE_DIR)
|
||||||
|
.join(bucket)
|
||||||
|
.join(ts)
|
||||||
|
.join(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn atomic_swap(src: &Path, dst: &Path) -> std::io::Result<()> {
|
||||||
|
if let Some(parent) = dst.parent() {
|
||||||
|
std::fs::create_dir_all(parent)?;
|
||||||
|
}
|
||||||
|
std::fs::rename(src, dst)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_stored_etag(detail: &str) -> String {
|
||||||
|
detail
|
||||||
|
.split_whitespace()
|
||||||
|
.find_map(|s| s.strip_prefix("stored_etag="))
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_actual_etag(detail: &str) -> String {
|
||||||
|
detail
|
||||||
|
.split_whitespace()
|
||||||
|
.find_map(|s| s.strip_prefix("actual_etag="))
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_result_json(
|
||||||
|
state: ScanState,
|
||||||
|
heal_stats: BTreeMap<String, HealStats>,
|
||||||
|
elapsed: f64,
|
||||||
|
) -> Value {
|
||||||
|
let issues_healed: u64 = heal_stats.values().map(|s| s.healed).sum();
|
||||||
|
let heal_stats_json: serde_json::Map<String, Value> = heal_stats
|
||||||
|
.iter()
|
||||||
|
.map(|(k, v)| (k.clone(), v.to_value()))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
json!({
|
||||||
|
"objects_scanned": state.objects_scanned,
|
||||||
|
"buckets_scanned": state.buckets_scanned,
|
||||||
|
"corrupted_objects": state.corrupted_objects,
|
||||||
|
"orphaned_objects": state.orphaned_objects,
|
||||||
|
"phantom_metadata": state.phantom_metadata,
|
||||||
|
"stale_versions": state.stale_versions,
|
||||||
|
"etag_cache_inconsistencies": state.etag_cache_inconsistencies,
|
||||||
|
"issues_healed": issues_healed,
|
||||||
|
"heal_stats": Value::Object(heal_stats_json),
|
||||||
|
"issues": state.issues,
|
||||||
|
"errors": state.errors,
|
||||||
|
"execution_time_seconds": elapsed,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn scan_all_buckets(storage_root: &Path, batch_size: usize) -> ScanState {
|
fn scan_all_buckets(storage_root: &Path, batch_size: usize) -> ScanState {
|
||||||
let mut state = ScanState::default();
|
let mut state = ScanState::default();
|
||||||
let buckets = match list_bucket_names(storage_root) {
|
let buckets = match list_bucket_names(storage_root) {
|
||||||
@@ -359,6 +854,18 @@ fn stored_etag(entry: &Value) -> Option<String> {
|
|||||||
.map(|s| s.to_string())
|
.map(|s| s.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn entry_metadata_map(entry: &Value) -> HashMap<String, String> {
|
||||||
|
entry
|
||||||
|
.get("metadata")
|
||||||
|
.and_then(|m| m.as_object())
|
||||||
|
.map(|m| {
|
||||||
|
m.iter()
|
||||||
|
.filter_map(|(k, v)| v.as_str().map(|s| (k.clone(), s.to_string())))
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
fn check_corrupted(
|
fn check_corrupted(
|
||||||
state: &mut ScanState,
|
state: &mut ScanState,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -378,12 +885,20 @@ fn check_corrupted(
|
|||||||
if !object_path.exists() {
|
if !object_path.exists() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
let meta_map = entry_metadata_map(&info.entry);
|
||||||
|
if metadata_is_corrupted(&meta_map) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
state.objects_scanned += 1;
|
state.objects_scanned += 1;
|
||||||
|
|
||||||
let Some(stored) = stored_etag(&info.entry) else {
|
let Some(stored) = stored_etag(&info.entry) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if is_multipart_etag(&stored) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
match myfsio_crypto::hashing::md5_file(&object_path) {
|
match myfsio_crypto::hashing::md5_file(&object_path) {
|
||||||
Ok(actual) => {
|
Ok(actual) => {
|
||||||
if actual != stored {
|
if actual != stored {
|
||||||
@@ -417,6 +932,10 @@ fn check_phantom(
|
|||||||
if state.batch_exhausted(batch_size) {
|
if state.batch_exhausted(batch_size) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
let info = &entries[full_key];
|
||||||
|
if metadata_is_corrupted(&entry_metadata_map(&info.entry)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
state.objects_scanned += 1;
|
state.objects_scanned += 1;
|
||||||
let object_path = bucket_path.join(full_key);
|
let object_path = bucket_path.join(full_key);
|
||||||
if !object_path.exists() {
|
if !object_path.exists() {
|
||||||
@@ -562,6 +1081,9 @@ fn check_stale_versions(
|
|||||||
}
|
}
|
||||||
state.objects_scanned += 1;
|
state.objects_scanned += 1;
|
||||||
if !bin_stems.contains_key(stem) {
|
if !bin_stems.contains_key(stem) {
|
||||||
|
if manifest_is_delete_marker(path) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
state.stale_versions += 1;
|
state.stale_versions += 1;
|
||||||
let key = path
|
let key = path
|
||||||
.strip_prefix(&versions_root)
|
.strip_prefix(&versions_root)
|
||||||
@@ -580,6 +1102,19 @@ fn check_stale_versions(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn manifest_is_delete_marker(path: &Path) -> bool {
|
||||||
|
let Ok(content) = std::fs::read_to_string(path) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let Ok(value) = serde_json::from_str::<Value>(&content) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
value
|
||||||
|
.get("is_delete_marker")
|
||||||
|
.and_then(Value::as_bool)
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
fn check_etag_cache(
|
fn check_etag_cache(
|
||||||
state: &mut ScanState,
|
state: &mut ScanState,
|
||||||
storage_root: &Path,
|
storage_root: &Path,
|
||||||
@@ -729,4 +1264,172 @@ mod tests {
|
|||||||
let state = scan_all_buckets(tmp.path(), 100);
|
let state = scan_all_buckets(tmp.path(), 100);
|
||||||
assert_eq!(state.buckets_scanned, 0);
|
assert_eq!(state.buckets_scanned, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn poisoned_entries_are_skipped_during_corruption_scan() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path();
|
||||||
|
let bucket = "testbucket";
|
||||||
|
let bucket_path = root.join(bucket);
|
||||||
|
let meta_root = root
|
||||||
|
.join(SYSTEM_ROOT)
|
||||||
|
.join(SYSTEM_BUCKETS_DIR)
|
||||||
|
.join(bucket)
|
||||||
|
.join(BUCKET_META_DIR);
|
||||||
|
fs::create_dir_all(&bucket_path).unwrap();
|
||||||
|
fs::create_dir_all(&meta_root).unwrap();
|
||||||
|
|
||||||
|
let bytes = b"some bytes that wont match";
|
||||||
|
fs::write(bucket_path.join("rotted.txt"), bytes).unwrap();
|
||||||
|
|
||||||
|
let mut map = Map::new();
|
||||||
|
map.insert(
|
||||||
|
"rotted.txt".to_string(),
|
||||||
|
json!({
|
||||||
|
"metadata": {
|
||||||
|
"__etag__": "00000000000000000000000000000000",
|
||||||
|
"__corrupted__": "true",
|
||||||
|
"__corruption_detail__": "etag mismatch (already poisoned)",
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
fs::write(
|
||||||
|
meta_root.join(INDEX_FILE),
|
||||||
|
serde_json::to_string(&Value::Object(map)).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let state = scan_all_buckets(root, 10_000);
|
||||||
|
assert_eq!(
|
||||||
|
state.corrupted_objects, 0,
|
||||||
|
"poisoned entries must not re-flag"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn delete_marker_manifests_are_not_flagged_stale() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path();
|
||||||
|
let bucket = "vbucket";
|
||||||
|
fs::create_dir_all(root.join(bucket)).unwrap();
|
||||||
|
|
||||||
|
let versions_dir = root
|
||||||
|
.join(SYSTEM_ROOT)
|
||||||
|
.join(SYSTEM_BUCKETS_DIR)
|
||||||
|
.join(bucket)
|
||||||
|
.join(BUCKET_VERSIONS_DIR)
|
||||||
|
.join("v.txt");
|
||||||
|
fs::create_dir_all(&versions_dir).unwrap();
|
||||||
|
|
||||||
|
let dm = json!({
|
||||||
|
"version_id": "dm-vid-1",
|
||||||
|
"key": "v.txt",
|
||||||
|
"size": 0,
|
||||||
|
"etag": "",
|
||||||
|
"is_delete_marker": true,
|
||||||
|
});
|
||||||
|
fs::write(
|
||||||
|
versions_dir.join("dm-vid-1.json"),
|
||||||
|
serde_json::to_string(&dm).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let truly_stale = json!({
|
||||||
|
"version_id": "broken-vid-2",
|
||||||
|
"key": "v.txt",
|
||||||
|
"size": 12,
|
||||||
|
"etag": "abc",
|
||||||
|
"is_delete_marker": false,
|
||||||
|
});
|
||||||
|
fs::write(
|
||||||
|
versions_dir.join("broken-vid-2.json"),
|
||||||
|
serde_json::to_string(&truly_stale).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let state = scan_all_buckets(root, 10_000);
|
||||||
|
assert_eq!(
|
||||||
|
state.stale_versions, 1,
|
||||||
|
"delete-marker manifest must not be flagged; only the data-bearing orphan should count"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_etag_helpers() {
|
||||||
|
let detail = "stored_etag=abc123 actual_etag=def456";
|
||||||
|
assert_eq!(parse_stored_etag(detail), "abc123");
|
||||||
|
assert_eq!(parse_actual_etag(detail), "def456");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn poisoned_entry_with_missing_file_is_not_phantom() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path();
|
||||||
|
let bucket = "testbucket";
|
||||||
|
let bucket_path = root.join(bucket);
|
||||||
|
let meta_root = root
|
||||||
|
.join(SYSTEM_ROOT)
|
||||||
|
.join(SYSTEM_BUCKETS_DIR)
|
||||||
|
.join(bucket)
|
||||||
|
.join(BUCKET_META_DIR);
|
||||||
|
fs::create_dir_all(&bucket_path).unwrap();
|
||||||
|
fs::create_dir_all(&meta_root).unwrap();
|
||||||
|
|
||||||
|
let mut map = Map::new();
|
||||||
|
map.insert(
|
||||||
|
"quarantined.txt".to_string(),
|
||||||
|
json!({
|
||||||
|
"metadata": {
|
||||||
|
"__etag__": "deadbeefdeadbeefdeadbeefdeadbeef",
|
||||||
|
"__corrupted__": "true",
|
||||||
|
"__corruption_detail__": "etag mismatch (no peer)",
|
||||||
|
"__quarantine_path__": ".myfsio.sys/quarantine/testbucket/2026/quarantined.txt",
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
fs::write(
|
||||||
|
meta_root.join(INDEX_FILE),
|
||||||
|
serde_json::to_string(&Value::Object(map)).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let state = scan_all_buckets(root, 10_000);
|
||||||
|
assert_eq!(
|
||||||
|
state.phantom_metadata, 0,
|
||||||
|
"poisoned entries with quarantined files must not be reported as phantom metadata"
|
||||||
|
);
|
||||||
|
assert_eq!(state.corrupted_objects, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn healthy_multipart_object_is_not_flagged_corrupted() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path();
|
||||||
|
let bucket = "testbucket";
|
||||||
|
let bucket_path = root.join(bucket);
|
||||||
|
let meta_root = root
|
||||||
|
.join(SYSTEM_ROOT)
|
||||||
|
.join(SYSTEM_BUCKETS_DIR)
|
||||||
|
.join(bucket)
|
||||||
|
.join(BUCKET_META_DIR);
|
||||||
|
fs::create_dir_all(&bucket_path).unwrap();
|
||||||
|
|
||||||
|
fs::write(bucket_path.join("multi.bin"), b"healthy multipart body").unwrap();
|
||||||
|
|
||||||
|
write_index(
|
||||||
|
&meta_root,
|
||||||
|
&[("multi.bin", "deadbeefdeadbeefdeadbeefdeadbeef-3")],
|
||||||
|
);
|
||||||
|
|
||||||
|
let state = scan_all_buckets(root, 10_000);
|
||||||
|
assert_eq!(
|
||||||
|
state.corrupted_objects, 0,
|
||||||
|
"multipart-style ETags must not be checked against whole-body MD5"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
state.errors.is_empty(),
|
||||||
|
"unexpected errors: {:?}",
|
||||||
|
state.errors
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ struct BucketLifecycleResult {
|
|||||||
struct ParsedLifecycleRule {
|
struct ParsedLifecycleRule {
|
||||||
status: String,
|
status: String,
|
||||||
prefix: String,
|
prefix: String,
|
||||||
|
tags: Vec<(String, String)>,
|
||||||
expiration_days: Option<u64>,
|
expiration_days: Option<u64>,
|
||||||
expiration_date: Option<DateTime<Utc>>,
|
expiration_date: Option<DateTime<Utc>>,
|
||||||
noncurrent_days: Option<u64>,
|
noncurrent_days: Option<u64>,
|
||||||
@@ -203,7 +204,9 @@ impl LifecycleService {
|
|||||||
match self.storage.list_objects(bucket, ¶ms).await {
|
match self.storage.list_objects(bucket, ¶ms).await {
|
||||||
Ok(objects) => {
|
Ok(objects) => {
|
||||||
for object in &objects.objects {
|
for object in &objects.objects {
|
||||||
if object.last_modified < cutoff {
|
if object.last_modified < cutoff
|
||||||
|
&& self.object_matches_tag_filter(bucket, &object.key, rule).await
|
||||||
|
{
|
||||||
if let Err(err) = self.storage.delete_object(bucket, &object.key).await {
|
if let Err(err) = self.storage.delete_object(bucket, &object.key).await {
|
||||||
result
|
result
|
||||||
.errors
|
.errors
|
||||||
@@ -219,6 +222,24 @@ impl LifecycleService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn object_matches_tag_filter(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
rule: &ParsedLifecycleRule,
|
||||||
|
) -> bool {
|
||||||
|
if rule.tags.is_empty() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
match self.storage.get_object_tags(bucket, key).await {
|
||||||
|
Ok(tags) => rule
|
||||||
|
.tags
|
||||||
|
.iter()
|
||||||
|
.all(|(k, v)| tags.iter().any(|t| t.key == *k && t.value == *v)),
|
||||||
|
Err(_) => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn apply_noncurrent_expiration_rule(
|
async fn apply_noncurrent_expiration_rule(
|
||||||
&self,
|
&self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -275,6 +296,21 @@ impl LifecycleService {
|
|||||||
if archived_at.is_none() || archived_at.unwrap() >= cutoff {
|
if archived_at.is_none() || archived_at.unwrap() >= cutoff {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if !rule.tags.is_empty() {
|
||||||
|
let Some(version_tags_value) = manifest.get("tags") else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let version_tags: Vec<myfsio_common::types::Tag> =
|
||||||
|
serde_json::from_value(version_tags_value.clone()).unwrap_or_default();
|
||||||
|
let matched = rule.tags.iter().all(|(k, v)| {
|
||||||
|
version_tags
|
||||||
|
.iter()
|
||||||
|
.any(|t| t.key == *k && t.value == *v)
|
||||||
|
});
|
||||||
|
if !matched {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
let version_id = manifest
|
let version_id = manifest
|
||||||
.get("version_id")
|
.get("version_id")
|
||||||
.and_then(|value| value.as_str())
|
.and_then(|value| value.as_str())
|
||||||
@@ -440,17 +476,49 @@ fn parse_lifecycle_rules_from_string(raw: &str) -> Vec<ParsedLifecycleRule> {
|
|||||||
status: child_text(&rule, "Status").unwrap_or_else(|| "Enabled".to_string()),
|
status: child_text(&rule, "Status").unwrap_or_else(|| "Enabled".to_string()),
|
||||||
prefix: child_text(&rule, "Prefix")
|
prefix: child_text(&rule, "Prefix")
|
||||||
.or_else(|| {
|
.or_else(|| {
|
||||||
rule.descendants()
|
let filter = rule.children().find(|node| {
|
||||||
.find(|node| {
|
node.is_element() && node.tag_name().name() == "Filter"
|
||||||
node.is_element()
|
})?;
|
||||||
&& node.tag_name().name() == "Filter"
|
if let Some(prefix) = child_text(&filter, "Prefix") {
|
||||||
&& node.children().any(|child| {
|
return Some(prefix);
|
||||||
child.is_element() && child.tag_name().name() == "Prefix"
|
}
|
||||||
})
|
let and = filter.children().find(|node| {
|
||||||
})
|
node.is_element() && node.tag_name().name() == "And"
|
||||||
.and_then(|filter| child_text(&filter, "Prefix"))
|
})?;
|
||||||
|
child_text(&and, "Prefix")
|
||||||
})
|
})
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
|
tags: {
|
||||||
|
let mut collected: Vec<(String, String)> = Vec::new();
|
||||||
|
if let Some(filter) = rule
|
||||||
|
.children()
|
||||||
|
.find(|node| node.is_element() && node.tag_name().name() == "Filter")
|
||||||
|
{
|
||||||
|
let direct_tag = filter
|
||||||
|
.children()
|
||||||
|
.find(|node| node.is_element() && node.tag_name().name() == "Tag");
|
||||||
|
if let Some(tag) = direct_tag {
|
||||||
|
if let Some(key) = child_text(&tag, "Key") {
|
||||||
|
collected.push((key, child_text(&tag, "Value").unwrap_or_default()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(and) = filter
|
||||||
|
.children()
|
||||||
|
.find(|node| node.is_element() && node.tag_name().name() == "And")
|
||||||
|
{
|
||||||
|
for tag in and
|
||||||
|
.children()
|
||||||
|
.filter(|node| node.is_element() && node.tag_name().name() == "Tag")
|
||||||
|
{
|
||||||
|
if let Some(key) = child_text(&tag, "Key") {
|
||||||
|
collected
|
||||||
|
.push((key, child_text(&tag, "Value").unwrap_or_default()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
collected
|
||||||
|
},
|
||||||
expiration_days: rule
|
expiration_days: rule
|
||||||
.descendants()
|
.descendants()
|
||||||
.find(|node| node.is_element() && node.tag_name().name() == "Expiration")
|
.find(|node| node.is_element() && node.tag_name().name() == "Expiration")
|
||||||
@@ -482,6 +550,37 @@ fn parse_lifecycle_rules_from_string(raw: &str) -> Vec<ParsedLifecycleRule> {
|
|||||||
|
|
||||||
fn parse_lifecycle_rule(value: &Value) -> Option<ParsedLifecycleRule> {
|
fn parse_lifecycle_rule(value: &Value) -> Option<ParsedLifecycleRule> {
|
||||||
let map = value.as_object()?;
|
let map = value.as_object()?;
|
||||||
|
let mut tags: Vec<(String, String)> = Vec::new();
|
||||||
|
if let Some(filter) = map.get("Filter").and_then(|v| v.as_object()) {
|
||||||
|
if let Some(tag) = filter.get("Tag").and_then(|v| v.as_object()) {
|
||||||
|
if let (Some(k), Some(v)) = (
|
||||||
|
tag.get("Key").and_then(|v| v.as_str()),
|
||||||
|
tag.get("Value").and_then(|v| v.as_str()),
|
||||||
|
) {
|
||||||
|
tags.push((k.to_string(), v.to_string()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(and) = filter.get("And").and_then(|v| v.as_object()) {
|
||||||
|
if let Some(arr) = and.get("Tags").and_then(|v| v.as_array()) {
|
||||||
|
for entry in arr {
|
||||||
|
if let (Some(k), Some(v)) = (
|
||||||
|
entry.get("Key").and_then(|v| v.as_str()),
|
||||||
|
entry.get("Value").and_then(|v| v.as_str()),
|
||||||
|
) {
|
||||||
|
tags.push((k.to_string(), v.to_string()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(tag) = and.get("Tag").and_then(|v| v.as_object()) {
|
||||||
|
if let (Some(k), Some(v)) = (
|
||||||
|
tag.get("Key").and_then(|v| v.as_str()),
|
||||||
|
tag.get("Value").and_then(|v| v.as_str()),
|
||||||
|
) {
|
||||||
|
tags.push((k.to_string(), v.to_string()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Some(ParsedLifecycleRule {
|
Some(ParsedLifecycleRule {
|
||||||
status: map
|
status: map
|
||||||
.get("Status")
|
.get("Status")
|
||||||
@@ -496,8 +595,15 @@ fn parse_lifecycle_rule(value: &Value) -> Option<ParsedLifecycleRule> {
|
|||||||
.and_then(|value| value.get("Prefix"))
|
.and_then(|value| value.get("Prefix"))
|
||||||
.and_then(|value| value.as_str())
|
.and_then(|value| value.as_str())
|
||||||
})
|
})
|
||||||
|
.or_else(|| {
|
||||||
|
map.get("Filter")
|
||||||
|
.and_then(|value| value.get("And"))
|
||||||
|
.and_then(|value| value.get("Prefix"))
|
||||||
|
.and_then(|value| value.as_str())
|
||||||
|
})
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
.to_string(),
|
.to_string(),
|
||||||
|
tags,
|
||||||
expiration_days: map
|
expiration_days: map
|
||||||
.get("Expiration")
|
.get("Expiration")
|
||||||
.and_then(|value| value.get("Days"))
|
.and_then(|value| value.get("Days"))
|
||||||
@@ -568,6 +674,74 @@ mod tests {
|
|||||||
assert_eq!(rules[0].abort_incomplete_multipart_days, Some(7));
|
assert_eq!(rules[0].abort_incomplete_multipart_days, Some(7));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_xml_filter_and_with_prefix_and_tags() {
|
||||||
|
let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<LifecycleConfiguration>
|
||||||
|
<Rule>
|
||||||
|
<Status>Enabled</Status>
|
||||||
|
<Filter>
|
||||||
|
<And>
|
||||||
|
<Prefix>logs/</Prefix>
|
||||||
|
<Tag><Key>env</Key><Value>prod</Value></Tag>
|
||||||
|
<Tag><Key>tier</Key><Value>cold</Value></Tag>
|
||||||
|
</And>
|
||||||
|
</Filter>
|
||||||
|
<Expiration><Days>10</Days></Expiration>
|
||||||
|
</Rule>
|
||||||
|
</LifecycleConfiguration>"#;
|
||||||
|
let rules = parse_lifecycle_rules(&Value::String(xml.to_string()));
|
||||||
|
assert_eq!(rules.len(), 1);
|
||||||
|
assert_eq!(rules[0].prefix, "logs/");
|
||||||
|
assert_eq!(
|
||||||
|
rules[0].tags,
|
||||||
|
vec![
|
||||||
|
("env".to_string(), "prod".to_string()),
|
||||||
|
("tier".to_string(), "cold".to_string()),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(rules[0].expiration_days, Some(10));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_xml_filter_with_single_tag() {
|
||||||
|
let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<LifecycleConfiguration>
|
||||||
|
<Rule>
|
||||||
|
<Status>Enabled</Status>
|
||||||
|
<Filter><Tag><Key>env</Key><Value>prod</Value></Tag></Filter>
|
||||||
|
<Expiration><Days>5</Days></Expiration>
|
||||||
|
</Rule>
|
||||||
|
</LifecycleConfiguration>"#;
|
||||||
|
let rules = parse_lifecycle_rules(&Value::String(xml.to_string()));
|
||||||
|
assert_eq!(rules.len(), 1);
|
||||||
|
assert_eq!(rules[0].prefix, "");
|
||||||
|
assert_eq!(
|
||||||
|
rules[0].tags,
|
||||||
|
vec![("env".to_string(), "prod".to_string())]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn xml_tags_outside_filter_are_ignored() {
|
||||||
|
// a stray <Tag> nested under an action must not be picked up as a filter tag
|
||||||
|
let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<LifecycleConfiguration>
|
||||||
|
<Rule>
|
||||||
|
<Status>Enabled</Status>
|
||||||
|
<Filter><Prefix>logs/</Prefix></Filter>
|
||||||
|
<Expiration>
|
||||||
|
<Days>10</Days>
|
||||||
|
<Tag><Key>spurious</Key><Value>nope</Value></Tag>
|
||||||
|
</Expiration>
|
||||||
|
</Rule>
|
||||||
|
</LifecycleConfiguration>"#;
|
||||||
|
let rules = parse_lifecycle_rules(&Value::String(xml.to_string()));
|
||||||
|
assert_eq!(rules.len(), 1);
|
||||||
|
assert!(rules[0].tags.is_empty());
|
||||||
|
assert_eq!(rules[0].prefix, "logs/");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn run_cycle_writes_history_and_deletes_noncurrent_versions() {
|
async fn run_cycle_writes_history_and_deletes_noncurrent_versions() {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
@@ -634,4 +808,156 @@ mod tests {
|
|||||||
assert_eq!(history["total"], 1);
|
assert_eq!(history["total"], 1);
|
||||||
assert_eq!(history["executions"][0]["versions_deleted"], 1);
|
assert_eq!(history["executions"][0]["versions_deleted"], 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn noncurrent_tag_filter_uses_version_tags_not_current_tags() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let storage = Arc::new(FsStorageBackend::new(tmp.path().to_path_buf()));
|
||||||
|
storage.create_bucket("docs").await.unwrap();
|
||||||
|
storage.set_versioning("docs", true).await.unwrap();
|
||||||
|
|
||||||
|
storage
|
||||||
|
.put_object(
|
||||||
|
"docs",
|
||||||
|
"logs/file.txt",
|
||||||
|
Box::pin(std::io::Cursor::new(b"v1".to_vec())),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
storage
|
||||||
|
.put_object(
|
||||||
|
"docs",
|
||||||
|
"logs/file.txt",
|
||||||
|
Box::pin(std::io::Cursor::new(b"v2".to_vec())),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let live_tags = vec![myfsio_common::types::Tag {
|
||||||
|
key: "env".to_string(),
|
||||||
|
value: "prod".to_string(),
|
||||||
|
}];
|
||||||
|
storage
|
||||||
|
.set_object_tags("docs", "logs/file.txt", &live_tags)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let versions_root = version_root_for_bucket(tmp.path(), "docs")
|
||||||
|
.join("logs")
|
||||||
|
.join("file.txt");
|
||||||
|
let manifest = std::fs::read_dir(&versions_root)
|
||||||
|
.unwrap()
|
||||||
|
.flatten()
|
||||||
|
.find(|entry| entry.path().extension().and_then(|ext| ext.to_str()) == Some("json"))
|
||||||
|
.unwrap()
|
||||||
|
.path();
|
||||||
|
let archived_manifest = json!({
|
||||||
|
"version_id": "ver-untagged",
|
||||||
|
"key": "logs/file.txt",
|
||||||
|
"size": 2,
|
||||||
|
"archived_at": (Utc::now() - Duration::days(45)).to_rfc3339(),
|
||||||
|
"etag": "etag",
|
||||||
|
"tags": [],
|
||||||
|
});
|
||||||
|
std::fs::write(
|
||||||
|
&manifest,
|
||||||
|
serde_json::to_string(&archived_manifest).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
std::fs::write(manifest.with_file_name("ver-untagged.bin"), b"v1").unwrap();
|
||||||
|
|
||||||
|
let lifecycle_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<LifecycleConfiguration>
|
||||||
|
<Rule>
|
||||||
|
<Status>Enabled</Status>
|
||||||
|
<Filter><And><Prefix>logs/</Prefix><Tag><Key>env</Key><Value>prod</Value></Tag></And></Filter>
|
||||||
|
<NoncurrentVersionExpiration><NoncurrentDays>30</NoncurrentDays></NoncurrentVersionExpiration>
|
||||||
|
</Rule>
|
||||||
|
</LifecycleConfiguration>"#;
|
||||||
|
let mut config = storage.get_bucket_config("docs").await.unwrap();
|
||||||
|
config.lifecycle = Some(Value::String(lifecycle_xml.to_string()));
|
||||||
|
storage.set_bucket_config("docs", &config).await.unwrap();
|
||||||
|
|
||||||
|
let service =
|
||||||
|
LifecycleService::new(storage.clone(), tmp.path(), LifecycleConfig::default());
|
||||||
|
let result = service.run_cycle().await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result["versions_deleted"], 0,
|
||||||
|
"noncurrent expiration must consult the version's own tags, not the current key's"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
manifest.exists(),
|
||||||
|
"the untagged archived version should still be on disk"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn noncurrent_tag_filter_deletes_when_version_tags_match() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let storage = Arc::new(FsStorageBackend::new(tmp.path().to_path_buf()));
|
||||||
|
storage.create_bucket("docs").await.unwrap();
|
||||||
|
storage.set_versioning("docs", true).await.unwrap();
|
||||||
|
|
||||||
|
storage
|
||||||
|
.put_object(
|
||||||
|
"docs",
|
||||||
|
"logs/file.txt",
|
||||||
|
Box::pin(std::io::Cursor::new(b"v1".to_vec())),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
storage
|
||||||
|
.put_object(
|
||||||
|
"docs",
|
||||||
|
"logs/file.txt",
|
||||||
|
Box::pin(std::io::Cursor::new(b"v2".to_vec())),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let versions_root = version_root_for_bucket(tmp.path(), "docs")
|
||||||
|
.join("logs")
|
||||||
|
.join("file.txt");
|
||||||
|
let manifest = std::fs::read_dir(&versions_root)
|
||||||
|
.unwrap()
|
||||||
|
.flatten()
|
||||||
|
.find(|entry| entry.path().extension().and_then(|ext| ext.to_str()) == Some("json"))
|
||||||
|
.unwrap()
|
||||||
|
.path();
|
||||||
|
let archived_manifest = json!({
|
||||||
|
"version_id": "ver-tagged",
|
||||||
|
"key": "logs/file.txt",
|
||||||
|
"size": 2,
|
||||||
|
"archived_at": (Utc::now() - Duration::days(45)).to_rfc3339(),
|
||||||
|
"etag": "etag",
|
||||||
|
"tags": [{ "key": "env", "value": "prod" }],
|
||||||
|
});
|
||||||
|
std::fs::write(
|
||||||
|
&manifest,
|
||||||
|
serde_json::to_string(&archived_manifest).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
std::fs::write(manifest.with_file_name("ver-tagged.bin"), b"v1").unwrap();
|
||||||
|
|
||||||
|
let lifecycle_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<LifecycleConfiguration>
|
||||||
|
<Rule>
|
||||||
|
<Status>Enabled</Status>
|
||||||
|
<Filter><And><Prefix>logs/</Prefix><Tag><Key>env</Key><Value>prod</Value></Tag></And></Filter>
|
||||||
|
<NoncurrentVersionExpiration><NoncurrentDays>30</NoncurrentDays></NoncurrentVersionExpiration>
|
||||||
|
</Rule>
|
||||||
|
</LifecycleConfiguration>"#;
|
||||||
|
let mut config = storage.get_bucket_config("docs").await.unwrap();
|
||||||
|
config.lifecycle = Some(Value::String(lifecycle_xml.to_string()));
|
||||||
|
storage.set_bucket_config("docs", &config).await.unwrap();
|
||||||
|
|
||||||
|
let service =
|
||||||
|
LifecycleService::new(storage.clone(), tmp.path(), LifecycleConfig::default());
|
||||||
|
let result = service.run_cycle().await.unwrap();
|
||||||
|
assert_eq!(result["versions_deleted"], 1);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -319,10 +319,11 @@ impl MetricsService {
|
|||||||
let _ = std::fs::create_dir_all(parent);
|
let _ = std::fs::create_dir_all(parent);
|
||||||
}
|
}
|
||||||
let data = json!({ "snapshots": snapshots });
|
let data = json!({ "snapshots": snapshots });
|
||||||
let _ = std::fs::write(
|
let serialized = serde_json::to_string_pretty(&data).unwrap_or_default();
|
||||||
&self.snapshots_path,
|
let tmp = self.snapshots_path.with_extension("json.tmp");
|
||||||
serde_json::to_string_pretty(&data).unwrap_or_default(),
|
if std::fs::write(&tmp, serialized).is_ok() {
|
||||||
);
|
let _ = std::fs::rename(&tmp, &self.snapshots_path);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn start_background(self: Arc<Self>) -> tokio::task::JoinHandle<()> {
|
pub fn start_background(self: Arc<Self>) -> tokio::task::JoinHandle<()> {
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ pub mod lifecycle;
|
|||||||
pub mod metrics;
|
pub mod metrics;
|
||||||
pub mod notifications;
|
pub mod notifications;
|
||||||
pub mod object_lock;
|
pub mod object_lock;
|
||||||
|
pub mod peer_admin;
|
||||||
|
pub mod peer_fetch;
|
||||||
pub mod replication;
|
pub mod replication;
|
||||||
pub mod s3_client;
|
pub mod s3_client;
|
||||||
pub mod site_registry;
|
pub mod site_registry;
|
||||||
|
|||||||
@@ -65,6 +65,24 @@ pub fn parse_notification_configurations(
|
|||||||
xml: &str,
|
xml: &str,
|
||||||
) -> Result<Vec<NotificationConfiguration>, String> {
|
) -> Result<Vec<NotificationConfiguration>, String> {
|
||||||
let doc = roxmltree::Document::parse(xml).map_err(|err| err.to_string())?;
|
let doc = roxmltree::Document::parse(xml).map_err(|err| err.to_string())?;
|
||||||
|
|
||||||
|
for unsupported in [
|
||||||
|
"TopicConfiguration",
|
||||||
|
"QueueConfiguration",
|
||||||
|
"CloudFunctionConfiguration",
|
||||||
|
"LambdaFunctionConfiguration",
|
||||||
|
] {
|
||||||
|
if doc
|
||||||
|
.descendants()
|
||||||
|
.any(|node| node.is_element() && node.tag_name().name() == unsupported)
|
||||||
|
{
|
||||||
|
return Err(format!(
|
||||||
|
"{} is not supported on this server; only WebhookConfiguration is accepted",
|
||||||
|
unsupported
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let mut configs = Vec::new();
|
let mut configs = Vec::new();
|
||||||
|
|
||||||
for webhook in doc
|
for webhook in doc
|
||||||
|
|||||||
183
crates/myfsio-server/src/services/peer_admin.rs
Normal file
183
crates/myfsio-server/src/services/peer_admin.rs
Normal file
@@ -0,0 +1,183 @@
|
|||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use chrono::Utc;
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
fn extract_error_detail(body: &str) -> String {
|
||||||
|
let trimmed = body.trim();
|
||||||
|
if trimmed.is_empty() {
|
||||||
|
return String::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Ok(value) = serde_json::from_str::<Value>(trimmed) {
|
||||||
|
let err = value.get("error").unwrap_or(&value);
|
||||||
|
let code = err
|
||||||
|
.get("code")
|
||||||
|
.or_else(|| err.get("Code"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty());
|
||||||
|
let message = err
|
||||||
|
.get("message")
|
||||||
|
.or_else(|| err.get("Message"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty());
|
||||||
|
let detail = match (code, message) {
|
||||||
|
(Some(c), Some(m)) => format!("{}: {}", c, m),
|
||||||
|
(Some(c), None) => c.to_string(),
|
||||||
|
(None, Some(m)) => m.to_string(),
|
||||||
|
(None, None) => String::new(),
|
||||||
|
};
|
||||||
|
if !detail.is_empty() {
|
||||||
|
return truncate_chars(&detail, 240);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let collapsed = trimmed
|
||||||
|
.lines()
|
||||||
|
.map(|l| l.trim())
|
||||||
|
.filter(|l| !l.is_empty())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" ");
|
||||||
|
truncate_chars(&collapsed, 240)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn truncate_chars(s: &str, max_chars: usize) -> String {
|
||||||
|
match s.char_indices().nth(max_chars) {
|
||||||
|
Some((boundary, _)) => format!("{}…", &s[..boundary]),
|
||||||
|
None => s.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
use myfsio_auth::sigv4::{
|
||||||
|
aws_uri_encode, build_string_to_sign, compute_signature, derive_signing_key, sha256_hex,
|
||||||
|
};
|
||||||
|
|
||||||
|
use crate::stores::connections::RemoteConnection;
|
||||||
|
|
||||||
|
pub struct PeerAdminClient {
|
||||||
|
client: reqwest::Client,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PeerAdminClient {
|
||||||
|
pub fn new(connect_timeout: Duration, read_timeout: Duration) -> Self {
|
||||||
|
let client = reqwest::Client::builder()
|
||||||
|
.connect_timeout(connect_timeout)
|
||||||
|
.timeout(read_timeout)
|
||||||
|
.build()
|
||||||
|
.unwrap_or_else(|_| reqwest::Client::new());
|
||||||
|
Self { client }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn fetch_cluster_overview(
|
||||||
|
&self,
|
||||||
|
endpoint: &str,
|
||||||
|
connection: &RemoteConnection,
|
||||||
|
) -> Result<Value, String> {
|
||||||
|
let url = format!(
|
||||||
|
"{}/admin/cluster/overview",
|
||||||
|
endpoint.trim_end_matches('/')
|
||||||
|
);
|
||||||
|
let parsed = reqwest::Url::parse(&url).map_err(|e| format!("invalid url: {}", e))?;
|
||||||
|
let host = parsed
|
||||||
|
.host_str()
|
||||||
|
.ok_or_else(|| "missing host".to_string())?
|
||||||
|
.to_string();
|
||||||
|
let host_with_port = match parsed.port() {
|
||||||
|
Some(p) => format!("{}:{}", host, p),
|
||||||
|
None => host.clone(),
|
||||||
|
};
|
||||||
|
let canonical_uri = parsed.path().to_string();
|
||||||
|
let canonical_uri = if canonical_uri.is_empty() {
|
||||||
|
"/".to_string()
|
||||||
|
} else {
|
||||||
|
canonical_uri
|
||||||
|
};
|
||||||
|
|
||||||
|
let now = Utc::now();
|
||||||
|
let amz_date = now.format("%Y%m%dT%H%M%SZ").to_string();
|
||||||
|
let date_stamp = now.format("%Y%m%d").to_string();
|
||||||
|
let region = if connection.region.is_empty() {
|
||||||
|
"us-east-1".to_string()
|
||||||
|
} else {
|
||||||
|
connection.region.clone()
|
||||||
|
};
|
||||||
|
let service = "s3";
|
||||||
|
let payload_hash = sha256_hex(b"");
|
||||||
|
|
||||||
|
let canonical_headers = format!(
|
||||||
|
"host:{}\nx-amz-content-sha256:{}\nx-amz-date:{}\n",
|
||||||
|
host_with_port, payload_hash, amz_date
|
||||||
|
);
|
||||||
|
let signed_headers = "host;x-amz-content-sha256;x-amz-date";
|
||||||
|
|
||||||
|
let canonical_query = parsed
|
||||||
|
.query()
|
||||||
|
.map(|q| {
|
||||||
|
let mut pairs: Vec<(String, String)> = q
|
||||||
|
.split('&')
|
||||||
|
.filter(|p| !p.is_empty())
|
||||||
|
.map(|p| {
|
||||||
|
let mut it = p.splitn(2, '=');
|
||||||
|
let k = it.next().unwrap_or("").to_string();
|
||||||
|
let v = it.next().unwrap_or("").to_string();
|
||||||
|
(k, v)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
pairs.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)));
|
||||||
|
pairs
|
||||||
|
.iter()
|
||||||
|
.map(|(k, v)| format!("{}={}", aws_uri_encode(k), aws_uri_encode(v)))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("&")
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let canonical_request = format!(
|
||||||
|
"GET\n{}\n{}\n{}\n{}\n{}",
|
||||||
|
canonical_uri, canonical_query, canonical_headers, signed_headers, payload_hash
|
||||||
|
);
|
||||||
|
|
||||||
|
let credential_scope = format!("{}/{}/{}/aws4_request", date_stamp, region, service);
|
||||||
|
let string_to_sign = build_string_to_sign(&amz_date, &credential_scope, &canonical_request);
|
||||||
|
let signing_key =
|
||||||
|
derive_signing_key(&connection.secret_key, &date_stamp, ®ion, service);
|
||||||
|
let signature = compute_signature(&signing_key, &string_to_sign);
|
||||||
|
|
||||||
|
let authorization = format!(
|
||||||
|
"AWS4-HMAC-SHA256 Credential={}/{},SignedHeaders={},Signature={}",
|
||||||
|
connection.access_key, credential_scope, signed_headers, signature
|
||||||
|
);
|
||||||
|
|
||||||
|
let resp = self
|
||||||
|
.client
|
||||||
|
.get(&url)
|
||||||
|
.header("host", &host_with_port)
|
||||||
|
.header("x-amz-content-sha256", &payload_hash)
|
||||||
|
.header("x-amz-date", &amz_date)
|
||||||
|
.header("authorization", &authorization)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("request failed: {}", e))?;
|
||||||
|
|
||||||
|
let status = resp.status();
|
||||||
|
if !status.is_success() {
|
||||||
|
let body_text = resp.text().await.unwrap_or_default();
|
||||||
|
let detail = extract_error_detail(&body_text);
|
||||||
|
if detail.is_empty() {
|
||||||
|
return Err(format!("peer returned status {}", status.as_u16()));
|
||||||
|
}
|
||||||
|
return Err(format!(
|
||||||
|
"peer returned status {} — {}",
|
||||||
|
status.as_u16(),
|
||||||
|
detail
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let body: Value = resp
|
||||||
|
.json()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("invalid json: {}", e))?;
|
||||||
|
Ok(body)
|
||||||
|
}
|
||||||
|
}
|
||||||
385
crates/myfsio-server/src/services/peer_fetch.rs
Normal file
385
crates/myfsio-server/src/services/peer_fetch.rs
Normal file
@@ -0,0 +1,385 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::Path;
|
||||||
|
use std::pin::Pin;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use aws_sdk_s3::Client;
|
||||||
|
use md5::{Digest, Md5};
|
||||||
|
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt};
|
||||||
|
|
||||||
|
use myfsio_storage::fs_backend::{is_multipart_etag, FsStorageBackend};
|
||||||
|
use myfsio_storage::traits::StorageEngine;
|
||||||
|
|
||||||
|
use crate::services::replication::ReplicationManager;
|
||||||
|
use crate::services::s3_client::{build_client, ClientOptions};
|
||||||
|
use crate::stores::connections::ConnectionStore;
|
||||||
|
|
||||||
|
pub struct PeerFetcher {
|
||||||
|
storage: Arc<FsStorageBackend>,
|
||||||
|
connections: Arc<ConnectionStore>,
|
||||||
|
replication: Arc<ReplicationManager>,
|
||||||
|
client_options: ClientOptions,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub enum HealOutcome {
|
||||||
|
Healed { peer_etag: String, bytes: u64 },
|
||||||
|
PeerMismatch { stored: String, peer: String },
|
||||||
|
PeerUnavailable { error: String },
|
||||||
|
NotConfigured,
|
||||||
|
VerifyFailed { expected: String, actual: String },
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PeerFetcher {
|
||||||
|
pub fn new(
|
||||||
|
storage: Arc<FsStorageBackend>,
|
||||||
|
connections: Arc<ConnectionStore>,
|
||||||
|
replication: Arc<ReplicationManager>,
|
||||||
|
client_options: ClientOptions,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
storage,
|
||||||
|
connections,
|
||||||
|
replication,
|
||||||
|
client_options,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_client_for_bucket(&self, bucket: &str) -> Option<(Client, String)> {
|
||||||
|
let rule = self.replication.get_rule(bucket)?;
|
||||||
|
if !rule.enabled {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let conn = self.connections.get(&rule.target_connection_id)?;
|
||||||
|
let client = build_client(&conn, &self.client_options);
|
||||||
|
Some((client, rule.target_bucket))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn fetch_into_storage(
|
||||||
|
&self,
|
||||||
|
client: &Client,
|
||||||
|
remote_bucket: &str,
|
||||||
|
local_bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
) -> bool {
|
||||||
|
let resp = match client
|
||||||
|
.get_object()
|
||||||
|
.bucket(remote_bucket)
|
||||||
|
.key(key)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(err) => {
|
||||||
|
tracing::error!("Pull GetObject failed {}/{}: {:?}", local_bucket, key, err);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let head = match client
|
||||||
|
.head_object()
|
||||||
|
.bucket(remote_bucket)
|
||||||
|
.key(key)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(err) => {
|
||||||
|
tracing::error!("Pull HeadObject failed {}/{}: {:?}", local_bucket, key, err);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let metadata: Option<HashMap<String, String>> = head
|
||||||
|
.metadata()
|
||||||
|
.map(|m| m.iter().map(|(k, v)| (k.clone(), v.clone())).collect());
|
||||||
|
|
||||||
|
let stream = resp.body.into_async_read();
|
||||||
|
let boxed: Pin<Box<dyn AsyncRead + Send>> = Box::pin(stream);
|
||||||
|
|
||||||
|
match self
|
||||||
|
.storage
|
||||||
|
.put_object(local_bucket, key, boxed, metadata)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(_) => {
|
||||||
|
tracing::debug!("Pulled object {}/{} from remote", local_bucket, key);
|
||||||
|
true
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
tracing::error!(
|
||||||
|
"Store pulled object failed {}/{}: {}",
|
||||||
|
local_bucket,
|
||||||
|
key,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn fetch_for_heal(
|
||||||
|
&self,
|
||||||
|
local_bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
expected_etag: &str,
|
||||||
|
dest_path: &Path,
|
||||||
|
) -> HealOutcome {
|
||||||
|
let (client, target_bucket) = match self.build_client_for_bucket(local_bucket) {
|
||||||
|
Some(v) => v,
|
||||||
|
None => return HealOutcome::NotConfigured,
|
||||||
|
};
|
||||||
|
|
||||||
|
let head = match client
|
||||||
|
.head_object()
|
||||||
|
.bucket(&target_bucket)
|
||||||
|
.key(key)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(err) => {
|
||||||
|
return HealOutcome::PeerUnavailable {
|
||||||
|
error: format!("HeadObject: {:?}", err),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let peer_etag = head.e_tag().unwrap_or("").trim_matches('"').to_string();
|
||||||
|
if peer_etag.is_empty() {
|
||||||
|
return HealOutcome::PeerUnavailable {
|
||||||
|
error: "remote returned empty ETag".into(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if peer_etag != expected_etag {
|
||||||
|
return HealOutcome::PeerMismatch {
|
||||||
|
stored: expected_etag.to_string(),
|
||||||
|
peer: peer_etag,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if is_multipart_etag(expected_etag) {
|
||||||
|
return self
|
||||||
|
.fetch_multipart_for_heal(&client, &target_bucket, key, expected_etag, dest_path)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
let resp = match client
|
||||||
|
.get_object()
|
||||||
|
.bucket(&target_bucket)
|
||||||
|
.key(key)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(err) => {
|
||||||
|
return HealOutcome::PeerUnavailable {
|
||||||
|
error: format!("GetObject: {:?}", err),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some(parent) = dest_path.parent() {
|
||||||
|
if let Err(e) = tokio::fs::create_dir_all(parent).await {
|
||||||
|
return HealOutcome::PeerUnavailable {
|
||||||
|
error: format!("mkdir parent: {}", e),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut file = match tokio::fs::File::create(dest_path).await {
|
||||||
|
Ok(f) => f,
|
||||||
|
Err(e) => {
|
||||||
|
return HealOutcome::PeerUnavailable {
|
||||||
|
error: format!("create temp: {}", e),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let mut reader = resp.body.into_async_read();
|
||||||
|
let mut hasher = Md5::new();
|
||||||
|
let mut buf = vec![0u8; 64 * 1024];
|
||||||
|
let mut total: u64 = 0;
|
||||||
|
loop {
|
||||||
|
let n = match reader.read(&mut buf).await {
|
||||||
|
Ok(n) => n,
|
||||||
|
Err(e) => {
|
||||||
|
drop(file);
|
||||||
|
let _ = tokio::fs::remove_file(dest_path).await;
|
||||||
|
return HealOutcome::PeerUnavailable {
|
||||||
|
error: format!("read body: {}", e),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if n == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
hasher.update(&buf[..n]);
|
||||||
|
if let Err(e) = file.write_all(&buf[..n]).await {
|
||||||
|
drop(file);
|
||||||
|
let _ = tokio::fs::remove_file(dest_path).await;
|
||||||
|
return HealOutcome::PeerUnavailable {
|
||||||
|
error: format!("write temp: {}", e),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
total += n as u64;
|
||||||
|
}
|
||||||
|
if let Err(e) = file.flush().await {
|
||||||
|
return HealOutcome::PeerUnavailable {
|
||||||
|
error: format!("flush temp: {}", e),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
drop(file);
|
||||||
|
|
||||||
|
let actual = format!("{:x}", hasher.finalize());
|
||||||
|
if actual != expected_etag {
|
||||||
|
let _ = tokio::fs::remove_file(dest_path).await;
|
||||||
|
return HealOutcome::VerifyFailed {
|
||||||
|
expected: expected_etag.to_string(),
|
||||||
|
actual,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
HealOutcome::Healed {
|
||||||
|
peer_etag,
|
||||||
|
bytes: total,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch_multipart_for_heal(
|
||||||
|
&self,
|
||||||
|
client: &Client,
|
||||||
|
target_bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
expected_etag: &str,
|
||||||
|
dest_path: &Path,
|
||||||
|
) -> HealOutcome {
|
||||||
|
let part_count = match expected_etag
|
||||||
|
.split_once('-')
|
||||||
|
.and_then(|(_, n)| n.parse::<u32>().ok())
|
||||||
|
{
|
||||||
|
Some(n) if n >= 1 => n,
|
||||||
|
_ => {
|
||||||
|
return HealOutcome::VerifyFailed {
|
||||||
|
expected: expected_etag.to_string(),
|
||||||
|
actual: format!("unparseable multipart suffix in {}", expected_etag),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some(parent) = dest_path.parent() {
|
||||||
|
if let Err(e) = tokio::fs::create_dir_all(parent).await {
|
||||||
|
return HealOutcome::PeerUnavailable {
|
||||||
|
error: format!("mkdir parent: {}", e),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut file = match tokio::fs::File::create(dest_path).await {
|
||||||
|
Ok(f) => f,
|
||||||
|
Err(e) => {
|
||||||
|
return HealOutcome::PeerUnavailable {
|
||||||
|
error: format!("create temp: {}", e),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut composite = Md5::new();
|
||||||
|
let mut total: u64 = 0;
|
||||||
|
let mut buf = vec![0u8; 64 * 1024];
|
||||||
|
|
||||||
|
for part_no in 1..=part_count {
|
||||||
|
let part_no_i32 = part_no as i32;
|
||||||
|
let resp = match client
|
||||||
|
.get_object()
|
||||||
|
.bucket(target_bucket)
|
||||||
|
.key(key)
|
||||||
|
.part_number(part_no_i32)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(err) => {
|
||||||
|
drop(file);
|
||||||
|
let _ = tokio::fs::remove_file(dest_path).await;
|
||||||
|
return HealOutcome::PeerUnavailable {
|
||||||
|
error: format!("GetObject part {}: {:?}", part_no, err),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut reader = resp.body.into_async_read();
|
||||||
|
let mut part_hasher = Md5::new();
|
||||||
|
let mut part_bytes: u64 = 0;
|
||||||
|
loop {
|
||||||
|
let n = match reader.read(&mut buf).await {
|
||||||
|
Ok(n) => n,
|
||||||
|
Err(e) => {
|
||||||
|
drop(file);
|
||||||
|
let _ = tokio::fs::remove_file(dest_path).await;
|
||||||
|
return HealOutcome::PeerUnavailable {
|
||||||
|
error: format!("read part {}: {}", part_no, e),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if n == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
part_hasher.update(&buf[..n]);
|
||||||
|
if let Err(e) = file.write_all(&buf[..n]).await {
|
||||||
|
drop(file);
|
||||||
|
let _ = tokio::fs::remove_file(dest_path).await;
|
||||||
|
return HealOutcome::PeerUnavailable {
|
||||||
|
error: format!("write part {}: {}", part_no, e),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
part_bytes += n as u64;
|
||||||
|
}
|
||||||
|
if part_bytes == 0 {
|
||||||
|
drop(file);
|
||||||
|
let _ = tokio::fs::remove_file(dest_path).await;
|
||||||
|
return HealOutcome::VerifyFailed {
|
||||||
|
expected: expected_etag.to_string(),
|
||||||
|
actual: format!("part {} returned zero bytes", part_no),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
composite.update(part_hasher.finalize().as_slice());
|
||||||
|
total += part_bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Err(e) = file.flush().await {
|
||||||
|
return HealOutcome::PeerUnavailable {
|
||||||
|
error: format!("flush temp: {}", e),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
drop(file);
|
||||||
|
|
||||||
|
let composite_etag = format!("{:x}-{}", composite.finalize(), part_count);
|
||||||
|
if composite_etag != expected_etag {
|
||||||
|
let _ = tokio::fs::remove_file(dest_path).await;
|
||||||
|
return HealOutcome::VerifyFailed {
|
||||||
|
expected: expected_etag.to_string(),
|
||||||
|
actual: composite_etag,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
HealOutcome::Healed {
|
||||||
|
peer_etag: expected_etag.to_string(),
|
||||||
|
bytes: total,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use myfsio_storage::fs_backend::is_multipart_etag;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn detects_multipart_etags() {
|
||||||
|
assert!(is_multipart_etag("d41d8cd98f00b204e9800998ecf8427e-3"));
|
||||||
|
assert!(is_multipart_etag("00000000000000000000000000000000-1"));
|
||||||
|
assert!(!is_multipart_etag("d41d8cd98f00b204e9800998ecf8427e"));
|
||||||
|
assert!(!is_multipart_etag("d41d8cd98f00b204e9800998ecf8427e-"));
|
||||||
|
assert!(!is_multipart_etag("not-hex-at-all-1"));
|
||||||
|
assert!(!is_multipart_etag("d41d8cd98f00b204e9800998ecf8427e-abc"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,10 +9,13 @@ use serde::{Deserialize, Serialize};
|
|||||||
use tokio::sync::Semaphore;
|
use tokio::sync::Semaphore;
|
||||||
|
|
||||||
use myfsio_common::types::ListParams;
|
use myfsio_common::types::ListParams;
|
||||||
use myfsio_storage::fs_backend::FsStorageBackend;
|
use myfsio_storage::fs_backend::{metadata_is_corrupted, FsStorageBackend};
|
||||||
use myfsio_storage::traits::StorageEngine;
|
use myfsio_storage::traits::StorageEngine;
|
||||||
|
|
||||||
use crate::services::s3_client::{build_client, check_endpoint_health, ClientOptions};
|
use crate::services::s3_client::{
|
||||||
|
build_client, build_health_client, check_endpoint_health, check_target_bucket_reachable,
|
||||||
|
ClientOptions,
|
||||||
|
};
|
||||||
use crate::stores::connections::{ConnectionStore, RemoteConnection};
|
use crate::stores::connections::{ConnectionStore, RemoteConnection};
|
||||||
|
|
||||||
pub const MODE_NEW_ONLY: &str = "new_only";
|
pub const MODE_NEW_ONLY: &str = "new_only";
|
||||||
@@ -347,7 +350,10 @@ impl ReplicationManager {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if !self.check_endpoint(&connection).await {
|
if !self
|
||||||
|
.check_target_bucket(&connection, &rule.target_bucket)
|
||||||
|
.await
|
||||||
|
{
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
"Cannot replicate existing objects for {}: endpoint {} is unreachable",
|
"Cannot replicate existing objects for {}: endpoint {} is unreachable",
|
||||||
bucket,
|
bucket,
|
||||||
@@ -459,6 +465,7 @@ impl ReplicationManager {
|
|||||||
self.failures.remove(bucket, object_key);
|
self.failures.remove(bucket, object_key);
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
|
let code = sdk_error_code(&err);
|
||||||
let msg = format!("{:?}", err);
|
let msg = format!("{:?}", err);
|
||||||
tracing::error!(
|
tracing::error!(
|
||||||
"Replication DELETE failed {}/{}: {}",
|
"Replication DELETE failed {}/{}: {}",
|
||||||
@@ -475,7 +482,7 @@ impl ReplicationManager {
|
|||||||
failure_count: 1,
|
failure_count: 1,
|
||||||
bucket_name: bucket.to_string(),
|
bucket_name: bucket.to_string(),
|
||||||
action: "delete".to_string(),
|
action: "delete".to_string(),
|
||||||
last_error_code: None,
|
last_error_code: code,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -483,6 +490,17 @@ impl ReplicationManager {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if let Ok(src_meta) = self.storage.get_object_metadata(bucket, object_key).await {
|
||||||
|
if metadata_is_corrupted(&src_meta) {
|
||||||
|
tracing::warn!(
|
||||||
|
"Replication skipped for {}/{}: source object is poisoned (corrupted)",
|
||||||
|
bucket,
|
||||||
|
object_key
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let src_path = match self.storage.get_object_path(bucket, object_key).await {
|
let src_path = match self.storage.get_object_path(bucket, object_key).await {
|
||||||
Ok(p) => p,
|
Ok(p) => p,
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
@@ -494,9 +512,39 @@ impl ReplicationManager {
|
|||||||
Ok(m) => m.len(),
|
Ok(m) => m.len(),
|
||||||
Err(_) => 0,
|
Err(_) => 0,
|
||||||
};
|
};
|
||||||
let content_type = mime_guess::from_path(&src_path)
|
let stored_meta = self
|
||||||
.first_raw()
|
.storage
|
||||||
.map(|s| s.to_string());
|
.get_object_metadata(bucket, object_key)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
let mut obj_meta = ReplicationObjectMeta::from_internal_metadata(&stored_meta);
|
||||||
|
if obj_meta.content_type.is_none() {
|
||||||
|
obj_meta.content_type = mime_guess::from_path(&src_path)
|
||||||
|
.first_raw()
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
}
|
||||||
|
if let Ok(tags) = self.storage.get_object_tags(bucket, object_key).await {
|
||||||
|
if !tags.is_empty() {
|
||||||
|
obj_meta.tagging_header = Some(
|
||||||
|
tags.iter()
|
||||||
|
.map(|t| {
|
||||||
|
format!(
|
||||||
|
"{}={}",
|
||||||
|
percent_encoding::utf8_percent_encode(
|
||||||
|
&t.key,
|
||||||
|
percent_encoding::NON_ALPHANUMERIC,
|
||||||
|
),
|
||||||
|
percent_encoding::utf8_percent_encode(
|
||||||
|
&t.value,
|
||||||
|
percent_encoding::NON_ALPHANUMERIC,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("&"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let upload_result = upload_object(
|
let upload_result = upload_object(
|
||||||
&client,
|
&client,
|
||||||
@@ -505,7 +553,7 @@ impl ReplicationManager {
|
|||||||
&src_path,
|
&src_path,
|
||||||
file_size,
|
file_size,
|
||||||
self.streaming_threshold_bytes,
|
self.streaming_threshold_bytes,
|
||||||
content_type.as_deref(),
|
Some(&obj_meta),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
@@ -529,7 +577,7 @@ impl ReplicationManager {
|
|||||||
&src_path,
|
&src_path,
|
||||||
file_size,
|
file_size,
|
||||||
self.streaming_threshold_bytes,
|
self.streaming_threshold_bytes,
|
||||||
content_type.as_deref(),
|
Some(&obj_meta),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
@@ -551,6 +599,7 @@ impl ReplicationManager {
|
|||||||
self.failures.remove(bucket, object_key);
|
self.failures.remove(bucket, object_key);
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
|
let code = upload_error_code(&err);
|
||||||
let msg = err.to_string();
|
let msg = err.to_string();
|
||||||
tracing::error!("Replication failed {}/{}: {}", bucket, object_key, msg);
|
tracing::error!("Replication failed {}/{}: {}", bucket, object_key, msg);
|
||||||
self.failures.add(
|
self.failures.add(
|
||||||
@@ -562,7 +611,7 @@ impl ReplicationManager {
|
|||||||
failure_count: 1,
|
failure_count: 1,
|
||||||
bucket_name: bucket.to_string(),
|
bucket_name: bucket.to_string(),
|
||||||
action: action.to_string(),
|
action: action.to_string(),
|
||||||
last_error_code: None,
|
last_error_code: code,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -570,10 +619,15 @@ impl ReplicationManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn check_endpoint(&self, conn: &RemoteConnection) -> bool {
|
pub async fn check_endpoint(&self, conn: &RemoteConnection) -> bool {
|
||||||
let client = build_client(conn, &self.client_options);
|
let client = build_health_client(conn, &self.client_options);
|
||||||
check_endpoint_health(&client).await
|
check_endpoint_health(&client).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn check_target_bucket(&self, conn: &RemoteConnection, target_bucket: &str) -> bool {
|
||||||
|
let client = build_client(conn, &self.client_options);
|
||||||
|
check_target_bucket_reachable(&client, target_bucket).await
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn retry_failed(&self, bucket: &str, object_key: &str) -> bool {
|
pub async fn retry_failed(&self, bucket: &str, object_key: &str) -> bool {
|
||||||
let failure = match self.failures.get(bucket, object_key) {
|
let failure = match self.failures.get(bucket, object_key) {
|
||||||
Some(f) => f,
|
Some(f) => f,
|
||||||
@@ -587,6 +641,15 @@ impl ReplicationManager {
|
|||||||
Some(c) => c,
|
Some(c) => c,
|
||||||
None => return false,
|
None => return false,
|
||||||
};
|
};
|
||||||
|
if !self.check_target_bucket(&conn, &rule.target_bucket).await {
|
||||||
|
tracing::warn!(
|
||||||
|
"Cannot retry {}/{}: endpoint {} is not reachable",
|
||||||
|
bucket,
|
||||||
|
object_key,
|
||||||
|
conn.endpoint_url
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
self.replicate_task(bucket, object_key, &rule, &conn, &failure.action)
|
self.replicate_task(bucket, object_key, &rule, &conn, &failure.action)
|
||||||
.await;
|
.await;
|
||||||
true
|
true
|
||||||
@@ -605,6 +668,15 @@ impl ReplicationManager {
|
|||||||
Some(c) => c,
|
Some(c) => c,
|
||||||
None => return (0, failures.len()),
|
None => return (0, failures.len()),
|
||||||
};
|
};
|
||||||
|
if !self.check_target_bucket(&conn, &rule.target_bucket).await {
|
||||||
|
tracing::warn!(
|
||||||
|
"Cannot retry {} failure(s) in {}: endpoint {} is not reachable",
|
||||||
|
failures.len(),
|
||||||
|
bucket,
|
||||||
|
conn.endpoint_url
|
||||||
|
);
|
||||||
|
return (0, failures.len());
|
||||||
|
}
|
||||||
let mut submitted = 0;
|
let mut submitted = 0;
|
||||||
for failure in failures {
|
for failure in failures {
|
||||||
self.replicate_task(bucket, &failure.object_key, &rule, &conn, &failure.action)
|
self.replicate_task(bucket, &failure.object_key, &rule, &conn, &failure.action)
|
||||||
@@ -664,6 +736,65 @@ fn is_no_such_bucket<E: std::fmt::Debug>(err: &E) -> bool {
|
|||||||
text.contains("NoSuchBucket")
|
text.contains("NoSuchBucket")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn sdk_error_code<E, R>(err: &aws_sdk_s3::error::SdkError<E, R>) -> Option<String>
|
||||||
|
where
|
||||||
|
E: aws_sdk_s3::error::ProvideErrorMetadata,
|
||||||
|
{
|
||||||
|
if let aws_sdk_s3::error::SdkError::ServiceError(svc) = err {
|
||||||
|
if let Some(code) = svc.err().code() {
|
||||||
|
return Some(code.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn upload_error_code(
|
||||||
|
err: &aws_sdk_s3::error::SdkError<aws_sdk_s3::operation::put_object::PutObjectError>,
|
||||||
|
) -> Option<String> {
|
||||||
|
sdk_error_code(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default, Clone)]
|
||||||
|
pub struct ReplicationObjectMeta {
|
||||||
|
pub content_type: Option<String>,
|
||||||
|
pub content_encoding: Option<String>,
|
||||||
|
pub content_disposition: Option<String>,
|
||||||
|
pub content_language: Option<String>,
|
||||||
|
pub cache_control: Option<String>,
|
||||||
|
pub expires: Option<String>,
|
||||||
|
pub storage_class: Option<String>,
|
||||||
|
pub website_redirect_location: Option<String>,
|
||||||
|
pub user_metadata: HashMap<String, String>,
|
||||||
|
pub tagging_header: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ReplicationObjectMeta {
|
||||||
|
pub fn from_internal_metadata(meta: &HashMap<String, String>) -> Self {
|
||||||
|
let mut user_metadata = HashMap::new();
|
||||||
|
for (k, v) in meta {
|
||||||
|
if k.starts_with("__") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if k.starts_with("x-amz-") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
user_metadata.insert(k.clone(), v.clone());
|
||||||
|
}
|
||||||
|
Self {
|
||||||
|
content_type: meta.get("__content_type__").cloned(),
|
||||||
|
content_encoding: meta.get("__content_encoding__").cloned(),
|
||||||
|
content_disposition: meta.get("__content_disposition__").cloned(),
|
||||||
|
content_language: meta.get("__content_language__").cloned(),
|
||||||
|
cache_control: meta.get("__cache_control__").cloned(),
|
||||||
|
expires: meta.get("__expires__").cloned(),
|
||||||
|
storage_class: meta.get("__storage_class__").cloned(),
|
||||||
|
website_redirect_location: meta.get("__website_redirect_location__").cloned(),
|
||||||
|
user_metadata,
|
||||||
|
tagging_header: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn upload_object(
|
async fn upload_object(
|
||||||
client: &aws_sdk_s3::Client,
|
client: &aws_sdk_s3::Client,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -671,11 +802,44 @@ async fn upload_object(
|
|||||||
path: &Path,
|
path: &Path,
|
||||||
file_size: u64,
|
file_size: u64,
|
||||||
streaming_threshold: u64,
|
streaming_threshold: u64,
|
||||||
content_type: Option<&str>,
|
obj_meta: Option<&ReplicationObjectMeta>,
|
||||||
) -> Result<(), aws_sdk_s3::error::SdkError<aws_sdk_s3::operation::put_object::PutObjectError>> {
|
) -> Result<(), aws_sdk_s3::error::SdkError<aws_sdk_s3::operation::put_object::PutObjectError>> {
|
||||||
let mut req = client.put_object().bucket(bucket).key(key);
|
let mut req = client.put_object().bucket(bucket).key(key);
|
||||||
if let Some(ct) = content_type {
|
if let Some(meta) = obj_meta {
|
||||||
req = req.content_type(ct);
|
if let Some(ref ct) = meta.content_type {
|
||||||
|
req = req.content_type(ct);
|
||||||
|
}
|
||||||
|
if let Some(ref v) = meta.content_encoding {
|
||||||
|
req = req.content_encoding(v);
|
||||||
|
}
|
||||||
|
if let Some(ref v) = meta.content_disposition {
|
||||||
|
req = req.content_disposition(v);
|
||||||
|
}
|
||||||
|
if let Some(ref v) = meta.content_language {
|
||||||
|
req = req.content_language(v);
|
||||||
|
}
|
||||||
|
if let Some(ref v) = meta.cache_control {
|
||||||
|
req = req.cache_control(v);
|
||||||
|
}
|
||||||
|
if let Some(ref v) = meta.expires {
|
||||||
|
if let Ok(dt) = chrono::DateTime::parse_from_rfc2822(v) {
|
||||||
|
req = req.expires(aws_smithy_types::DateTime::from_secs(dt.timestamp()));
|
||||||
|
} else if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(v) {
|
||||||
|
req = req.expires(aws_smithy_types::DateTime::from_secs(dt.timestamp()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(ref v) = meta.storage_class {
|
||||||
|
req = req.storage_class(aws_sdk_s3::types::StorageClass::from(v.as_str()));
|
||||||
|
}
|
||||||
|
if let Some(ref v) = meta.website_redirect_location {
|
||||||
|
req = req.website_redirect_location(v);
|
||||||
|
}
|
||||||
|
if let Some(ref v) = meta.tagging_header {
|
||||||
|
req = req.tagging(v);
|
||||||
|
}
|
||||||
|
for (k, v) in &meta.user_metadata {
|
||||||
|
req = req.metadata(k, v);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let body = if file_size >= streaming_threshold {
|
let body = if file_size >= streaming_threshold {
|
||||||
|
|||||||
@@ -2,11 +2,13 @@ use std::time::Duration;
|
|||||||
|
|
||||||
use aws_config::BehaviorVersion;
|
use aws_config::BehaviorVersion;
|
||||||
use aws_credential_types::Credentials;
|
use aws_credential_types::Credentials;
|
||||||
use aws_sdk_s3::config::{Region, SharedCredentialsProvider};
|
use aws_sdk_s3::config::{AppName, Region, SharedCredentialsProvider};
|
||||||
use aws_sdk_s3::Client;
|
use aws_sdk_s3::Client;
|
||||||
|
|
||||||
use crate::stores::connections::RemoteConnection;
|
use crate::stores::connections::RemoteConnection;
|
||||||
|
|
||||||
|
pub const REPLICATION_USER_AGENT_TAG: &str = "MyFSIO-Replication";
|
||||||
|
|
||||||
pub struct ClientOptions {
|
pub struct ClientOptions {
|
||||||
pub connect_timeout: Duration,
|
pub connect_timeout: Duration,
|
||||||
pub read_timeout: Duration,
|
pub read_timeout: Duration,
|
||||||
@@ -40,17 +42,29 @@ pub fn build_client(connection: &RemoteConnection, options: &ClientOptions) -> C
|
|||||||
let retry_config =
|
let retry_config =
|
||||||
aws_smithy_types::retry::RetryConfig::standard().with_max_attempts(options.max_attempts);
|
aws_smithy_types::retry::RetryConfig::standard().with_max_attempts(options.max_attempts);
|
||||||
|
|
||||||
let config = aws_sdk_s3::config::Builder::new()
|
let mut builder = aws_sdk_s3::config::Builder::new()
|
||||||
.behavior_version(BehaviorVersion::latest())
|
.behavior_version(BehaviorVersion::latest())
|
||||||
.credentials_provider(SharedCredentialsProvider::new(credentials))
|
.credentials_provider(SharedCredentialsProvider::new(credentials))
|
||||||
.region(Region::new(connection.region.clone()))
|
.region(Region::new(connection.region.clone()))
|
||||||
.endpoint_url(connection.endpoint_url.clone())
|
.endpoint_url(connection.endpoint_url.clone())
|
||||||
.force_path_style(true)
|
.force_path_style(true)
|
||||||
.timeout_config(timeout_config)
|
.timeout_config(timeout_config)
|
||||||
.retry_config(retry_config)
|
.retry_config(retry_config);
|
||||||
.build();
|
|
||||||
|
|
||||||
Client::from_conf(config)
|
if let Ok(app_name) = AppName::new(REPLICATION_USER_AGENT_TAG) {
|
||||||
|
builder = builder.app_name(app_name);
|
||||||
|
}
|
||||||
|
|
||||||
|
Client::from_conf(builder.build())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn build_health_client(connection: &RemoteConnection, options: &ClientOptions) -> Client {
|
||||||
|
let fast_fail = ClientOptions {
|
||||||
|
connect_timeout: options.connect_timeout,
|
||||||
|
read_timeout: options.read_timeout,
|
||||||
|
max_attempts: 1,
|
||||||
|
};
|
||||||
|
build_client(connection, &fast_fail)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn check_endpoint_health(client: &Client) -> bool {
|
pub async fn check_endpoint_health(client: &Client) -> bool {
|
||||||
@@ -62,3 +76,28 @@ pub async fn check_endpoint_health(client: &Client) -> bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn check_target_bucket_reachable(client: &Client, target_bucket: &str) -> bool {
|
||||||
|
match client.head_bucket().bucket(target_bucket).send().await {
|
||||||
|
Ok(_) => true,
|
||||||
|
Err(err) => match &err {
|
||||||
|
aws_sdk_s3::error::SdkError::ServiceError(_)
|
||||||
|
| aws_sdk_s3::error::SdkError::ResponseError(_) => {
|
||||||
|
tracing::debug!(
|
||||||
|
"Target-bucket reachability probe for {} got a server response; treating as reachable: {:?}",
|
||||||
|
target_bucket,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
true
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
tracing::warn!(
|
||||||
|
"Target-bucket reachability probe for {} failed at transport layer: {:?}",
|
||||||
|
target_bucket,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -38,6 +38,8 @@ pub struct PeerSite {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub connection_id: Option<String>,
|
pub connection_id: Option<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
|
pub peer_inbound_access_key: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
pub created_at: Option<String>,
|
pub created_at: Option<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub is_healthy: bool,
|
pub is_healthy: bool,
|
||||||
@@ -145,4 +147,15 @@ impl SiteRegistry {
|
|||||||
drop(data);
|
drop(data);
|
||||||
self.save();
|
self.save();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn is_peer_inbound_access_key(&self, access_key: &str) -> bool {
|
||||||
|
if access_key.is_empty() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
self.data
|
||||||
|
.read()
|
||||||
|
.peers
|
||||||
|
.iter()
|
||||||
|
.any(|p| p.peer_inbound_access_key.as_deref() == Some(access_key))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,19 +1,18 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::pin::Pin;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
use aws_sdk_s3::Client;
|
use aws_sdk_s3::Client;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use tokio::io::AsyncRead;
|
|
||||||
use tokio::sync::Notify;
|
use tokio::sync::Notify;
|
||||||
|
|
||||||
use myfsio_common::types::{ListParams, ObjectMeta};
|
use myfsio_common::types::{ListParams, ObjectMeta};
|
||||||
use myfsio_storage::fs_backend::FsStorageBackend;
|
use myfsio_storage::fs_backend::FsStorageBackend;
|
||||||
use myfsio_storage::traits::StorageEngine;
|
use myfsio_storage::traits::StorageEngine;
|
||||||
|
|
||||||
|
use crate::services::peer_fetch::PeerFetcher;
|
||||||
use crate::services::replication::{ReplicationManager, ReplicationRule, MODE_BIDIRECTIONAL};
|
use crate::services::replication::{ReplicationManager, ReplicationRule, MODE_BIDIRECTIONAL};
|
||||||
use crate::services::s3_client::{build_client, ClientOptions};
|
use crate::services::s3_client::{build_client, ClientOptions};
|
||||||
use crate::stores::connections::ConnectionStore;
|
use crate::stores::connections::ConnectionStore;
|
||||||
@@ -33,7 +32,7 @@ pub struct SyncState {
|
|||||||
pub last_full_sync: Option<f64>,
|
pub last_full_sync: Option<f64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, Serialize)]
|
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||||
pub struct SiteSyncStats {
|
pub struct SiteSyncStats {
|
||||||
pub last_sync_at: Option<f64>,
|
pub last_sync_at: Option<f64>,
|
||||||
pub objects_pulled: u64,
|
pub objects_pulled: u64,
|
||||||
@@ -53,6 +52,7 @@ pub struct SiteSyncWorker {
|
|||||||
storage: Arc<FsStorageBackend>,
|
storage: Arc<FsStorageBackend>,
|
||||||
connections: Arc<ConnectionStore>,
|
connections: Arc<ConnectionStore>,
|
||||||
replication: Arc<ReplicationManager>,
|
replication: Arc<ReplicationManager>,
|
||||||
|
peer_fetcher: Arc<PeerFetcher>,
|
||||||
storage_root: PathBuf,
|
storage_root: PathBuf,
|
||||||
interval: Duration,
|
interval: Duration,
|
||||||
batch_size: usize,
|
batch_size: usize,
|
||||||
@@ -75,24 +75,41 @@ impl SiteSyncWorker {
|
|||||||
max_retries: u32,
|
max_retries: u32,
|
||||||
clock_skew_tolerance: f64,
|
clock_skew_tolerance: f64,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
let client_options = ClientOptions {
|
||||||
storage,
|
connect_timeout,
|
||||||
connections,
|
read_timeout,
|
||||||
replication,
|
max_attempts: max_retries,
|
||||||
storage_root,
|
};
|
||||||
interval: Duration::from_secs(interval_seconds),
|
let peer_fetcher = Arc::new(PeerFetcher::new(
|
||||||
batch_size,
|
storage.clone(),
|
||||||
clock_skew_tolerance,
|
connections.clone(),
|
||||||
client_options: ClientOptions {
|
replication.clone(),
|
||||||
|
ClientOptions {
|
||||||
connect_timeout,
|
connect_timeout,
|
||||||
read_timeout,
|
read_timeout,
|
||||||
max_attempts: max_retries,
|
max_attempts: max_retries,
|
||||||
},
|
},
|
||||||
bucket_stats: Mutex::new(HashMap::new()),
|
));
|
||||||
|
let bucket_stats = Mutex::new(load_stats(&storage_root));
|
||||||
|
Self {
|
||||||
|
storage,
|
||||||
|
connections,
|
||||||
|
replication,
|
||||||
|
peer_fetcher,
|
||||||
|
storage_root,
|
||||||
|
interval: Duration::from_secs(interval_seconds),
|
||||||
|
batch_size,
|
||||||
|
clock_skew_tolerance,
|
||||||
|
client_options,
|
||||||
|
bucket_stats,
|
||||||
shutdown: Arc::new(Notify::new()),
|
shutdown: Arc::new(Notify::new()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn peer_fetcher(&self) -> Arc<PeerFetcher> {
|
||||||
|
self.peer_fetcher.clone()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn shutdown(&self) {
|
pub fn shutdown(&self) {
|
||||||
self.shutdown.notify_waiters();
|
self.shutdown.notify_waiters();
|
||||||
}
|
}
|
||||||
@@ -101,6 +118,15 @@ impl SiteSyncWorker {
|
|||||||
self.bucket_stats.lock().get(bucket).cloned()
|
self.bucket_stats.lock().get(bucket).cloned()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn snapshot_stats(&self) -> HashMap<String, SiteSyncStats> {
|
||||||
|
self.bucket_stats.lock().clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn save_stats(&self) {
|
||||||
|
let snapshot = self.bucket_stats.lock().clone();
|
||||||
|
save_stats(&self.storage_root, &snapshot);
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn run(self: Arc<Self>) {
|
pub async fn run(self: Arc<Self>) {
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
"Site sync worker started (interval={}s)",
|
"Site sync worker started (interval={}s)",
|
||||||
@@ -120,6 +146,7 @@ impl SiteSyncWorker {
|
|||||||
|
|
||||||
async fn run_cycle(&self) {
|
async fn run_cycle(&self) {
|
||||||
let rules = self.replication.rules_snapshot();
|
let rules = self.replication.rules_snapshot();
|
||||||
|
let mut mutated = false;
|
||||||
for (bucket, rule) in rules {
|
for (bucket, rule) in rules {
|
||||||
if rule.mode != MODE_BIDIRECTIONAL || !rule.enabled {
|
if rule.mode != MODE_BIDIRECTIONAL || !rule.enabled {
|
||||||
continue;
|
continue;
|
||||||
@@ -127,12 +154,16 @@ impl SiteSyncWorker {
|
|||||||
match self.sync_bucket(&rule).await {
|
match self.sync_bucket(&rule).await {
|
||||||
Ok(stats) => {
|
Ok(stats) => {
|
||||||
self.bucket_stats.lock().insert(bucket, stats);
|
self.bucket_stats.lock().insert(bucket, stats);
|
||||||
|
mutated = true;
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::error!("Site sync failed for bucket {}: {}", bucket, e);
|
tracing::error!("Site sync failed for bucket {}: {}", bucket, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if mutated {
|
||||||
|
self.save_stats();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn trigger_sync(&self, bucket: &str) -> Option<SiteSyncStats> {
|
pub async fn trigger_sync(&self, bucket: &str) -> Option<SiteSyncStats> {
|
||||||
@@ -145,6 +176,7 @@ impl SiteSyncWorker {
|
|||||||
self.bucket_stats
|
self.bucket_stats
|
||||||
.lock()
|
.lock()
|
||||||
.insert(bucket.to_string(), stats.clone());
|
.insert(bucket.to_string(), stats.clone());
|
||||||
|
self.save_stats();
|
||||||
Some(stats)
|
Some(stats)
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
@@ -383,60 +415,9 @@ impl SiteSyncWorker {
|
|||||||
local_bucket: &str,
|
local_bucket: &str,
|
||||||
key: &str,
|
key: &str,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
let resp = match client
|
self.peer_fetcher
|
||||||
.get_object()
|
.fetch_into_storage(client, remote_bucket, local_bucket, key)
|
||||||
.bucket(remote_bucket)
|
|
||||||
.key(key)
|
|
||||||
.send()
|
|
||||||
.await
|
.await
|
||||||
{
|
|
||||||
Ok(r) => r,
|
|
||||||
Err(err) => {
|
|
||||||
tracing::error!("Pull GetObject failed {}/{}: {:?}", local_bucket, key, err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let head = match client
|
|
||||||
.head_object()
|
|
||||||
.bucket(remote_bucket)
|
|
||||||
.key(key)
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(r) => r,
|
|
||||||
Err(err) => {
|
|
||||||
tracing::error!("Pull HeadObject failed {}/{}: {:?}", local_bucket, key, err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let metadata: Option<HashMap<String, String>> = head
|
|
||||||
.metadata()
|
|
||||||
.map(|m| m.iter().map(|(k, v)| (k.clone(), v.clone())).collect());
|
|
||||||
|
|
||||||
let stream = resp.body.into_async_read();
|
|
||||||
let boxed: Pin<Box<dyn AsyncRead + Send>> = Box::pin(stream);
|
|
||||||
|
|
||||||
match self
|
|
||||||
.storage
|
|
||||||
.put_object(local_bucket, key, boxed, metadata)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(_) => {
|
|
||||||
tracing::debug!("Pulled object {}/{} from remote", local_bucket, key);
|
|
||||||
true
|
|
||||||
}
|
|
||||||
Err(err) => {
|
|
||||||
tracing::error!(
|
|
||||||
"Store pulled object failed {}/{}: {}",
|
|
||||||
local_bucket,
|
|
||||||
key,
|
|
||||||
err
|
|
||||||
);
|
|
||||||
false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn apply_remote_deletion(&self, bucket: &str, key: &str) -> bool {
|
async fn apply_remote_deletion(&self, bucket: &str, key: &str) -> bool {
|
||||||
@@ -489,6 +470,34 @@ fn now_secs() -> f64 {
|
|||||||
.unwrap_or(0.0)
|
.unwrap_or(0.0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn stats_path(storage_root: &std::path::Path) -> PathBuf {
|
||||||
|
storage_root
|
||||||
|
.join(".myfsio.sys")
|
||||||
|
.join("config")
|
||||||
|
.join("site_sync_stats.json")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_stats(storage_root: &std::path::Path) -> HashMap<String, SiteSyncStats> {
|
||||||
|
let path = stats_path(storage_root);
|
||||||
|
if !path.exists() {
|
||||||
|
return HashMap::new();
|
||||||
|
}
|
||||||
|
match std::fs::read_to_string(&path) {
|
||||||
|
Ok(text) => serde_json::from_str(&text).unwrap_or_default(),
|
||||||
|
Err(_) => HashMap::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn save_stats(storage_root: &std::path::Path, stats: &HashMap<String, SiteSyncStats>) {
|
||||||
|
let path = stats_path(storage_root);
|
||||||
|
if let Some(parent) = path.parent() {
|
||||||
|
let _ = std::fs::create_dir_all(parent);
|
||||||
|
}
|
||||||
|
if let Ok(text) = serde_json::to_string_pretty(stats) {
|
||||||
|
let _ = std::fs::write(&path, text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn is_not_found_error<E: std::fmt::Debug>(err: &aws_sdk_s3::error::SdkError<E>) -> bool {
|
fn is_not_found_error<E: std::fmt::Debug>(err: &aws_sdk_s3::error::SdkError<E>) -> bool {
|
||||||
let msg = format!("{:?}", err);
|
let msg = format!("{:?}", err);
|
||||||
msg.contains("NoSuchBucket")
|
msg.contains("NoSuchBucket")
|
||||||
|
|||||||
@@ -144,7 +144,7 @@ fn normalize_path_for_mount(path: &Path) -> String {
|
|||||||
stripped.to_lowercase()
|
stripped.to_lowercase()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sample_disk(path: &Path) -> (u64, u64) {
|
pub fn sample_disk(path: &Path) -> (u64, u64) {
|
||||||
let disks = Disks::new_with_refreshed_list();
|
let disks = Disks::new_with_refreshed_list();
|
||||||
let path_str = normalize_path_for_mount(path);
|
let path_str = normalize_path_for_mount(path);
|
||||||
let mut best: Option<(usize, u64, u64)> = None;
|
let mut best: Option<(usize, u64, u64)> = None;
|
||||||
|
|||||||
@@ -1,12 +1,17 @@
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
use crate::config::ServerConfig;
|
use crate::config::ServerConfig;
|
||||||
use crate::services::access_logging::AccessLoggingService;
|
use crate::services::access_logging::AccessLoggingService;
|
||||||
use crate::services::gc::GcService;
|
use crate::services::gc::GcService;
|
||||||
use crate::services::integrity::IntegrityService;
|
use crate::services::integrity::IntegrityService;
|
||||||
use crate::services::metrics::MetricsService;
|
use crate::services::metrics::MetricsService;
|
||||||
|
use crate::services::peer_fetch::PeerFetcher;
|
||||||
use crate::services::replication::ReplicationManager;
|
use crate::services::replication::ReplicationManager;
|
||||||
|
use crate::services::s3_client::ClientOptions;
|
||||||
use crate::services::site_registry::SiteRegistry;
|
use crate::services::site_registry::SiteRegistry;
|
||||||
use crate::services::site_sync::SiteSyncWorker;
|
use crate::services::site_sync::SiteSyncWorker;
|
||||||
use crate::services::system_metrics::SystemMetricsService;
|
use crate::services::system_metrics::SystemMetricsService;
|
||||||
@@ -38,6 +43,8 @@ pub struct AppState {
|
|||||||
pub templates: Option<Arc<TemplateEngine>>,
|
pub templates: Option<Arc<TemplateEngine>>,
|
||||||
pub sessions: Arc<SessionStore>,
|
pub sessions: Arc<SessionStore>,
|
||||||
pub access_logging: Arc<AccessLoggingService>,
|
pub access_logging: Arc<AccessLoggingService>,
|
||||||
|
pub cluster_overview_cache: Arc<Mutex<Option<(Instant, Value)>>>,
|
||||||
|
pub cluster_aggregate_cache: Arc<Mutex<Option<(Instant, Value)>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AppState {
|
impl AppState {
|
||||||
@@ -50,6 +57,7 @@ impl AppState {
|
|||||||
bucket_config_cache_ttl: Duration::from_secs_f64(
|
bucket_config_cache_ttl: Duration::from_secs_f64(
|
||||||
config.bucket_config_cache_ttl_seconds,
|
config.bucket_config_cache_ttl_seconds,
|
||||||
),
|
),
|
||||||
|
stream_chunk_size: config.stream_chunk_size,
|
||||||
},
|
},
|
||||||
));
|
));
|
||||||
let iam = Arc::new(IamService::new_with_secret(
|
let iam = Arc::new(IamService::new_with_secret(
|
||||||
@@ -65,6 +73,7 @@ impl AppState {
|
|||||||
temp_file_max_age_hours: config.gc_temp_file_max_age_hours,
|
temp_file_max_age_hours: config.gc_temp_file_max_age_hours,
|
||||||
multipart_max_age_days: config.gc_multipart_max_age_days,
|
multipart_max_age_days: config.gc_multipart_max_age_days,
|
||||||
lock_file_max_age_hours: config.gc_lock_file_max_age_hours,
|
lock_file_max_age_hours: config.gc_lock_file_max_age_hours,
|
||||||
|
quarantine_max_age_days: config.integrity_quarantine_retention_days,
|
||||||
dry_run: config.gc_dry_run,
|
dry_run: config.gc_dry_run,
|
||||||
},
|
},
|
||||||
)))
|
)))
|
||||||
@@ -72,16 +81,6 @@ impl AppState {
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
let integrity = if config.integrity_enabled {
|
|
||||||
Some(Arc::new(IntegrityService::new(
|
|
||||||
storage.clone(),
|
|
||||||
&config.storage_root,
|
|
||||||
crate::services::integrity::IntegrityConfig::default(),
|
|
||||||
)))
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
|
|
||||||
let metrics = if config.metrics_enabled {
|
let metrics = if config.metrics_enabled {
|
||||||
Some(Arc::new(MetricsService::new(
|
Some(Arc::new(MetricsService::new(
|
||||||
&config.storage_root,
|
&config.storage_root,
|
||||||
@@ -160,7 +159,40 @@ impl AppState {
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
let templates = init_templates(&config.templates_dir);
|
let integrity_peer_fetcher: Option<Arc<PeerFetcher>> = if let Some(ref ss) = site_sync {
|
||||||
|
Some(ss.peer_fetcher())
|
||||||
|
} else {
|
||||||
|
Some(Arc::new(PeerFetcher::new(
|
||||||
|
storage.clone(),
|
||||||
|
connections.clone(),
|
||||||
|
replication.clone(),
|
||||||
|
ClientOptions {
|
||||||
|
connect_timeout: Duration::from_secs(config.site_sync_connect_timeout_secs),
|
||||||
|
read_timeout: Duration::from_secs(config.site_sync_read_timeout_secs),
|
||||||
|
max_attempts: config.site_sync_max_retries,
|
||||||
|
},
|
||||||
|
)))
|
||||||
|
};
|
||||||
|
|
||||||
|
let integrity = if config.integrity_enabled {
|
||||||
|
Some(Arc::new(IntegrityService::new(
|
||||||
|
storage.clone(),
|
||||||
|
&config.storage_root,
|
||||||
|
crate::services::integrity::IntegrityConfig {
|
||||||
|
interval_hours: config.integrity_interval_hours,
|
||||||
|
batch_size: config.integrity_batch_size,
|
||||||
|
auto_heal: config.integrity_auto_heal,
|
||||||
|
dry_run: config.integrity_dry_run,
|
||||||
|
heal_concurrency: config.integrity_heal_concurrency,
|
||||||
|
quarantine_retention_days: config.integrity_quarantine_retention_days,
|
||||||
|
},
|
||||||
|
integrity_peer_fetcher,
|
||||||
|
)))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
let templates = init_templates(&config.templates_dir, &config.display_timezone);
|
||||||
let access_logging = Arc::new(AccessLoggingService::new(&config.storage_root));
|
let access_logging = Arc::new(AccessLoggingService::new(&config.storage_root));
|
||||||
let session_ttl = Duration::from_secs(config.session_lifetime_days.saturating_mul(86_400));
|
let session_ttl = Duration::from_secs(config.session_lifetime_days.saturating_mul(86_400));
|
||||||
Self {
|
Self {
|
||||||
@@ -181,6 +213,8 @@ impl AppState {
|
|||||||
templates,
|
templates,
|
||||||
sessions: Arc::new(SessionStore::new(session_ttl)),
|
sessions: Arc::new(SessionStore::new(session_ttl)),
|
||||||
access_logging,
|
access_logging,
|
||||||
|
cluster_overview_cache: Arc::new(Mutex::new(None)),
|
||||||
|
cluster_aggregate_cache: Arc::new(Mutex::new(None)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -225,9 +259,20 @@ impl AppState {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn init_templates(templates_dir: &std::path::Path) -> Option<Arc<TemplateEngine>> {
|
fn init_templates(
|
||||||
let glob = format!("{}/*.html", templates_dir.display()).replace('\\', "/");
|
templates_dir: &std::path::Path,
|
||||||
match TemplateEngine::new(&glob) {
|
display_timezone: &str,
|
||||||
|
) -> Option<Arc<TemplateEngine>> {
|
||||||
|
let use_disk = std::env::var("TEMPLATES_DIR").is_ok() && templates_dir.is_dir();
|
||||||
|
let result = if use_disk {
|
||||||
|
let glob = format!("{}/*.html", templates_dir.display()).replace('\\', "/");
|
||||||
|
tracing::info!("Loading templates from disk: {}", templates_dir.display());
|
||||||
|
TemplateEngine::new(&glob, display_timezone)
|
||||||
|
} else {
|
||||||
|
tracing::info!("Loading templates from embedded assets");
|
||||||
|
TemplateEngine::from_embedded(display_timezone)
|
||||||
|
};
|
||||||
|
match result {
|
||||||
Ok(engine) => {
|
Ok(engine) => {
|
||||||
crate::handlers::ui_pages::register_ui_endpoints(&engine);
|
crate::handlers::ui_pages::register_ui_endpoints(&engine);
|
||||||
Some(Arc::new(engine))
|
Some(Arc::new(engine))
|
||||||
|
|||||||
@@ -1,9 +1,14 @@
|
|||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use base64::engine::general_purpose::URL_SAFE;
|
||||||
|
use base64::Engine;
|
||||||
use parking_lot::RwLock;
|
use parking_lot::RwLock;
|
||||||
|
use rand::RngCore;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
const ENCRYPTED_PREFIX: &str = "enc:";
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct RemoteConnection {
|
pub struct RemoteConnection {
|
||||||
pub id: String,
|
pub id: String,
|
||||||
@@ -21,6 +26,7 @@ fn default_region() -> String {
|
|||||||
|
|
||||||
pub struct ConnectionStore {
|
pub struct ConnectionStore {
|
||||||
path: PathBuf,
|
path: PathBuf,
|
||||||
|
encryption_key: String,
|
||||||
inner: Arc<RwLock<Vec<RemoteConnection>>>,
|
inner: Arc<RwLock<Vec<RemoteConnection>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -30,12 +36,17 @@ impl ConnectionStore {
|
|||||||
.join(".myfsio.sys")
|
.join(".myfsio.sys")
|
||||||
.join("config")
|
.join("config")
|
||||||
.join("connections.json");
|
.join("connections.json");
|
||||||
let inner = Arc::new(RwLock::new(load_from_disk(&path)));
|
let encryption_key = load_or_create_key(storage_root);
|
||||||
Self { path, inner }
|
let inner = Arc::new(RwLock::new(load_from_disk(&path, &encryption_key)));
|
||||||
|
Self {
|
||||||
|
path,
|
||||||
|
encryption_key,
|
||||||
|
inner,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn reload(&self) {
|
pub fn reload(&self) {
|
||||||
let loaded = load_from_disk(&self.path);
|
let loaded = load_from_disk(&self.path, &self.encryption_key);
|
||||||
*self.inner.write() = loaded;
|
*self.inner.write() = loaded;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -76,19 +87,76 @@ impl ConnectionStore {
|
|||||||
if let Some(parent) = self.path.parent() {
|
if let Some(parent) = self.path.parent() {
|
||||||
std::fs::create_dir_all(parent)?;
|
std::fs::create_dir_all(parent)?;
|
||||||
}
|
}
|
||||||
let snapshot = self.inner.read().clone();
|
let mut snapshot = self.inner.read().clone();
|
||||||
|
for conn in &mut snapshot {
|
||||||
|
if !conn.secret_key.starts_with(ENCRYPTED_PREFIX) {
|
||||||
|
if let Ok(token) =
|
||||||
|
myfsio_auth::fernet::encrypt(&self.encryption_key, conn.secret_key.as_bytes())
|
||||||
|
{
|
||||||
|
conn.secret_key = format!("{}{}", ENCRYPTED_PREFIX, token);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
let bytes = serde_json::to_vec_pretty(&snapshot)
|
let bytes = serde_json::to_vec_pretty(&snapshot)
|
||||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||||
std::fs::write(&self.path, bytes)
|
let tmp = self.path.with_extension("json.tmp");
|
||||||
|
std::fs::write(&tmp, bytes)?;
|
||||||
|
std::fs::rename(&tmp, &self.path)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn load_from_disk(path: &Path) -> Vec<RemoteConnection> {
|
fn load_or_create_key(storage_root: &Path) -> String {
|
||||||
|
let key_path = storage_root
|
||||||
|
.join(".myfsio.sys")
|
||||||
|
.join("config")
|
||||||
|
.join(".connections_key");
|
||||||
|
if let Ok(text) = std::fs::read_to_string(&key_path) {
|
||||||
|
let trimmed = text.trim();
|
||||||
|
if !trimmed.is_empty() {
|
||||||
|
if let Ok(decoded) = URL_SAFE.decode(trimmed) {
|
||||||
|
if decoded.len() == 32 {
|
||||||
|
return trimmed.to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut key = [0u8; 32];
|
||||||
|
rand::thread_rng().fill_bytes(&mut key);
|
||||||
|
let encoded = URL_SAFE.encode(key);
|
||||||
|
if let Some(parent) = key_path.parent() {
|
||||||
|
let _ = std::fs::create_dir_all(parent);
|
||||||
|
}
|
||||||
|
let _ = std::fs::write(&key_path, &encoded);
|
||||||
|
encoded
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_from_disk(path: &Path, encryption_key: &str) -> Vec<RemoteConnection> {
|
||||||
if !path.exists() {
|
if !path.exists() {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
}
|
}
|
||||||
match std::fs::read_to_string(path) {
|
let text = match std::fs::read_to_string(path) {
|
||||||
Ok(text) => serde_json::from_str(&text).unwrap_or_default(),
|
Ok(text) => text,
|
||||||
Err(_) => Vec::new(),
|
Err(_) => return Vec::new(),
|
||||||
|
};
|
||||||
|
let mut connections: Vec<RemoteConnection> =
|
||||||
|
serde_json::from_str(&text).unwrap_or_default();
|
||||||
|
for conn in &mut connections {
|
||||||
|
if let Some(token) = conn.secret_key.strip_prefix(ENCRYPTED_PREFIX) {
|
||||||
|
match myfsio_auth::fernet::decrypt(encryption_key, token) {
|
||||||
|
Ok(plaintext) => {
|
||||||
|
if let Ok(s) = String::from_utf8(plaintext) {
|
||||||
|
conn.secret_key = s;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
tracing::error!(
|
||||||
|
"Failed to decrypt peer secret_key for connection {}: {}",
|
||||||
|
conn.id,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
connections
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ use std::collections::HashMap;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
|
use chrono_tz::Tz;
|
||||||
use parking_lot::RwLock;
|
use parking_lot::RwLock;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use tera::{Context, Error as TeraError, Tera};
|
use tera::{Context, Error as TeraError, Tera};
|
||||||
@@ -16,10 +17,10 @@ pub struct TemplateEngine {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl TemplateEngine {
|
impl TemplateEngine {
|
||||||
pub fn new(template_glob: &str) -> Result<Self, TeraError> {
|
pub fn new(template_glob: &str, display_timezone: &str) -> Result<Self, TeraError> {
|
||||||
let mut tera = Tera::new(template_glob)?;
|
let mut tera = Tera::new(template_glob)?;
|
||||||
tera.set_escape_fn(html_escape);
|
tera.set_escape_fn(html_escape);
|
||||||
register_filters(&mut tera);
|
register_filters(&mut tera, display_timezone);
|
||||||
|
|
||||||
let endpoints: Arc<RwLock<HashMap<String, String>>> = Arc::new(RwLock::new(HashMap::new()));
|
let endpoints: Arc<RwLock<HashMap<String, String>>> = Arc::new(RwLock::new(HashMap::new()));
|
||||||
|
|
||||||
@@ -31,6 +32,33 @@ impl TemplateEngine {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn from_embedded(display_timezone: &str) -> Result<Self, TeraError> {
|
||||||
|
let mut tera = Tera::default();
|
||||||
|
tera.set_escape_fn(html_escape);
|
||||||
|
register_filters(&mut tera, display_timezone);
|
||||||
|
|
||||||
|
let names = crate::embedded::template_names();
|
||||||
|
let mut entries: Vec<(String, String)> = Vec::with_capacity(names.len());
|
||||||
|
for name in names {
|
||||||
|
if let Some(contents) = crate::embedded::template_contents(&name) {
|
||||||
|
entries.push((name, contents));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let refs: Vec<(&str, &str)> = entries
|
||||||
|
.iter()
|
||||||
|
.map(|(n, c)| (n.as_str(), c.as_str()))
|
||||||
|
.collect();
|
||||||
|
tera.add_raw_templates(refs)?;
|
||||||
|
|
||||||
|
let endpoints: Arc<RwLock<HashMap<String, String>>> = Arc::new(RwLock::new(HashMap::new()));
|
||||||
|
register_functions(&mut tera, endpoints.clone());
|
||||||
|
|
||||||
|
Ok(Self {
|
||||||
|
tera: Arc::new(RwLock::new(tera)),
|
||||||
|
endpoints,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn register_endpoint(&self, name: &str, path_template: &str) {
|
pub fn register_endpoint(&self, name: &str, path_template: &str) {
|
||||||
self.endpoints
|
self.endpoints
|
||||||
.write()
|
.write()
|
||||||
@@ -68,8 +96,14 @@ fn html_escape(input: &str) -> String {
|
|||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
fn register_filters(tera: &mut Tera) {
|
fn register_filters(tera: &mut Tera, display_timezone: &str) {
|
||||||
tera.register_filter("format_datetime", format_datetime_filter);
|
let tz: Tz = display_timezone.parse().unwrap_or(chrono_tz::UTC);
|
||||||
|
tera.register_filter(
|
||||||
|
"format_datetime",
|
||||||
|
move |value: &Value, args: &HashMap<String, Value>| -> tera::Result<Value> {
|
||||||
|
format_datetime_filter(value, args, tz)
|
||||||
|
},
|
||||||
|
);
|
||||||
tera.register_filter("filesizeformat", filesizeformat_filter);
|
tera.register_filter("filesizeformat", filesizeformat_filter);
|
||||||
tera.register_filter("slice", slice_filter);
|
tera.register_filter("slice", slice_filter);
|
||||||
}
|
}
|
||||||
@@ -159,11 +193,15 @@ fn urlencode_query(s: &str) -> String {
|
|||||||
percent_encoding::utf8_percent_encode(s, UNRESERVED).to_string()
|
percent_encoding::utf8_percent_encode(s, UNRESERVED).to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn format_datetime_filter(value: &Value, args: &HashMap<String, Value>) -> tera::Result<Value> {
|
fn format_datetime_filter(
|
||||||
|
value: &Value,
|
||||||
|
args: &HashMap<String, Value>,
|
||||||
|
tz: Tz,
|
||||||
|
) -> tera::Result<Value> {
|
||||||
let format = args
|
let format = args
|
||||||
.get("format")
|
.get("format")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.unwrap_or("%Y-%m-%d %H:%M:%S UTC");
|
.unwrap_or("%Y-%m-%d %H:%M:%S %Z");
|
||||||
|
|
||||||
let dt: Option<DateTime<Utc>> = match value {
|
let dt: Option<DateTime<Utc>> = match value {
|
||||||
Value::String(s) => DateTime::parse_from_rfc3339(s)
|
Value::String(s) => DateTime::parse_from_rfc3339(s)
|
||||||
@@ -183,7 +221,7 @@ fn format_datetime_filter(value: &Value, args: &HashMap<String, Value>) -> tera:
|
|||||||
};
|
};
|
||||||
|
|
||||||
match dt {
|
match dt {
|
||||||
Some(d) => Ok(Value::String(d.format(format).to_string())),
|
Some(d) => Ok(Value::String(d.with_timezone(&tz).format(format).to_string())),
|
||||||
None => Ok(value.clone()),
|
None => Ok(value.clone()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -264,7 +302,7 @@ mod tests {
|
|||||||
let tpl = tmp.path().join("t.html");
|
let tpl = tmp.path().join("t.html");
|
||||||
std::fs::write(&tpl, "").unwrap();
|
std::fs::write(&tpl, "").unwrap();
|
||||||
let glob = format!("{}/*.html", tmp.path().display());
|
let glob = format!("{}/*.html", tmp.path().display());
|
||||||
let engine = TemplateEngine::new(&glob).unwrap();
|
let engine = TemplateEngine::new(&glob, "UTC").unwrap();
|
||||||
engine.register_endpoints(&[
|
engine.register_endpoints(&[
|
||||||
("ui.buckets_overview", "/ui/buckets"),
|
("ui.buckets_overview", "/ui/buckets"),
|
||||||
("ui.bucket_detail", "/ui/buckets/{bucket_name}"),
|
("ui.bucket_detail", "/ui/buckets/{bucket_name}"),
|
||||||
@@ -329,7 +367,7 @@ mod tests {
|
|||||||
path.push("templates");
|
path.push("templates");
|
||||||
path.push("*.html");
|
path.push("*.html");
|
||||||
let glob = path.to_string_lossy().replace('\\', "/");
|
let glob = path.to_string_lossy().replace('\\', "/");
|
||||||
let engine = TemplateEngine::new(&glob).expect("Tera parse failed");
|
let engine = TemplateEngine::new(&glob, "UTC").expect("Tera parse failed");
|
||||||
let names: Vec<String> = engine
|
let names: Vec<String> = engine
|
||||||
.tera
|
.tera
|
||||||
.read()
|
.read()
|
||||||
@@ -343,13 +381,44 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn embedded_templates_parse() {
|
||||||
|
let engine = TemplateEngine::from_embedded("UTC").expect("Embedded Tera parse failed");
|
||||||
|
let names: Vec<String> = engine
|
||||||
|
.tera
|
||||||
|
.read()
|
||||||
|
.get_template_names()
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
.collect();
|
||||||
|
assert!(
|
||||||
|
names.len() >= 10,
|
||||||
|
"expected 10+ embedded templates, got {}",
|
||||||
|
names.len()
|
||||||
|
);
|
||||||
|
assert!(names.iter().any(|n| n == "login.html"));
|
||||||
|
assert!(names.iter().any(|n| n == "404.html"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn format_datetime_rfc3339() {
|
fn format_datetime_rfc3339() {
|
||||||
let v = format_datetime_filter(
|
let v = format_datetime_filter(
|
||||||
&Value::String("2024-06-15T12:34:56Z".into()),
|
&Value::String("2024-06-15T12:34:56Z".into()),
|
||||||
&HashMap::new(),
|
&HashMap::new(),
|
||||||
|
chrono_tz::UTC,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(v, Value::String("2024-06-15 12:34:56 UTC".into()));
|
assert_eq!(v, Value::String("2024-06-15 12:34:56 UTC".into()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn format_datetime_custom_timezone() {
|
||||||
|
let tz: Tz = "America/New_York".parse().unwrap();
|
||||||
|
let v = format_datetime_filter(
|
||||||
|
&Value::String("2024-06-15T12:34:56Z".into()),
|
||||||
|
&HashMap::new(),
|
||||||
|
tz,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(v, Value::String("2024-06-15 08:34:56 EDT".into()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -404,6 +404,7 @@ html.sidebar-will-collapse .sidebar-user {
|
|||||||
min-height: 70px;
|
min-height: 70px;
|
||||||
gap: 0.5rem;
|
gap: 0.5rem;
|
||||||
overflow: visible;
|
overflow: visible;
|
||||||
|
flex-shrink: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
.sidebar-collapsed .sidebar-header {
|
.sidebar-collapsed .sidebar-header {
|
||||||
@@ -516,10 +517,16 @@ html.sidebar-will-collapse .sidebar-user {
|
|||||||
}
|
}
|
||||||
|
|
||||||
.sidebar-body {
|
.sidebar-body {
|
||||||
flex: 1;
|
flex: 1 1 auto;
|
||||||
|
min-height: 0;
|
||||||
|
overflow-y: auto;
|
||||||
padding: 1rem 0;
|
padding: 1rem 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.sidebar-footer {
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
|
||||||
.sidebar-nav {
|
.sidebar-nav {
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
|
|||||||
@@ -336,6 +336,72 @@
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const renderObjectsLimit = (totalObjects, maxObjects) => {
|
||||||
|
if (maxObjects && maxObjects > 0) {
|
||||||
|
const pct = Math.min(100, Math.floor(totalObjects / maxObjects * 100));
|
||||||
|
const cls = pct >= 90 ? 'bg-danger' : pct >= 75 ? 'bg-warning' : 'bg-success';
|
||||||
|
return '<div class="progress mt-2" style="height: 4px;">' +
|
||||||
|
'<div class="progress-bar ' + cls + '" style="width: ' + pct + '%"></div>' +
|
||||||
|
'</div>' +
|
||||||
|
'<div class="small text-muted mt-1">' + pct + '% of ' + maxObjects.toLocaleString() + ' limit</div>';
|
||||||
|
}
|
||||||
|
return '<div class="small text-muted mt-2">No limit</div>';
|
||||||
|
};
|
||||||
|
|
||||||
|
const renderBytesLimit = (totalBytes, maxBytes) => {
|
||||||
|
if (maxBytes && maxBytes > 0) {
|
||||||
|
const pct = Math.min(100, Math.floor(totalBytes / maxBytes * 100));
|
||||||
|
const cls = pct >= 90 ? 'bg-danger' : pct >= 75 ? 'bg-warning' : 'bg-success';
|
||||||
|
return '<div class="progress mt-2" style="height: 4px;">' +
|
||||||
|
'<div class="progress-bar ' + cls + '" style="width: ' + pct + '%"></div>' +
|
||||||
|
'</div>' +
|
||||||
|
'<div class="small text-muted mt-1">' + pct + '% of ' + formatBytes(maxBytes) + ' limit</div>';
|
||||||
|
}
|
||||||
|
return '<div class="small text-muted mt-2">No limit</div>';
|
||||||
|
};
|
||||||
|
|
||||||
|
const redrawUsageLimits = () => {
|
||||||
|
const objectsCard = document.querySelector('[data-usage-objects]');
|
||||||
|
const objectsLimit = document.querySelector('[data-usage-objects-limit]');
|
||||||
|
if (objectsCard && objectsLimit) {
|
||||||
|
const totalObjects = parseInt(objectsCard.dataset.totalObjects || '0', 10);
|
||||||
|
const maxObjectsRaw = objectsCard.dataset.maxObjects;
|
||||||
|
const maxObjects = maxObjectsRaw ? parseInt(maxObjectsRaw, 10) : 0;
|
||||||
|
objectsLimit.innerHTML = renderObjectsLimit(totalObjects, maxObjects);
|
||||||
|
}
|
||||||
|
const bytesCard = document.querySelector('[data-usage-bytes]');
|
||||||
|
const bytesLimit = document.querySelector('[data-usage-bytes-limit]');
|
||||||
|
if (bytesCard && bytesLimit) {
|
||||||
|
const totalBytes = parseInt(bytesCard.dataset.totalBytes || '0', 10);
|
||||||
|
const maxBytesRaw = bytesCard.dataset.maxBytes;
|
||||||
|
const maxBytes = maxBytesRaw ? parseInt(maxBytesRaw, 10) : 0;
|
||||||
|
bytesLimit.innerHTML = renderBytesLimit(totalBytes, maxBytes);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const refreshBucketUsage = async () => {
|
||||||
|
try {
|
||||||
|
const bucketName = objectsContainer?.dataset.bucket;
|
||||||
|
if (!bucketName) return;
|
||||||
|
const url = `/ui/buckets/${encodeURIComponent(bucketName)}/stats`;
|
||||||
|
const response = await fetch(url, { headers: { 'Accept': 'application/json' } });
|
||||||
|
if (!response.ok) return;
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
const objectsCard = document.querySelector('[data-usage-objects]');
|
||||||
|
const objectsValue = document.querySelector('[data-usage-objects-value]');
|
||||||
|
if (objectsCard) objectsCard.dataset.totalObjects = String(data.total_objects);
|
||||||
|
if (objectsValue) objectsValue.textContent = data.total_objects.toLocaleString();
|
||||||
|
|
||||||
|
const bytesCard = document.querySelector('[data-usage-bytes]');
|
||||||
|
const bytesValue = document.querySelector('[data-usage-bytes-value]');
|
||||||
|
if (bytesCard) bytesCard.dataset.totalBytes = String(data.total_bytes);
|
||||||
|
if (bytesValue) bytesValue.textContent = formatBytes(data.total_bytes);
|
||||||
|
|
||||||
|
redrawUsageLimits();
|
||||||
|
} catch (e) { }
|
||||||
|
};
|
||||||
|
|
||||||
let topSpacer = null;
|
let topSpacer = null;
|
||||||
let bottomSpacer = null;
|
let bottomSpacer = null;
|
||||||
|
|
||||||
@@ -486,7 +552,13 @@
|
|||||||
let scrollTimeout = null;
|
let scrollTimeout = null;
|
||||||
const handleVirtualScroll = () => {
|
const handleVirtualScroll = () => {
|
||||||
if (scrollTimeout) cancelAnimationFrame(scrollTimeout);
|
if (scrollTimeout) cancelAnimationFrame(scrollTimeout);
|
||||||
scrollTimeout = requestAnimationFrame(renderVirtualRows);
|
scrollTimeout = requestAnimationFrame(() => {
|
||||||
|
renderVirtualRows();
|
||||||
|
const c = document.querySelector('.objects-table-container');
|
||||||
|
if (c && c.scrollHeight - c.scrollTop - c.clientHeight < 500) {
|
||||||
|
if (typeof loadMoreOnSentinel === 'function') loadMoreOnSentinel();
|
||||||
|
}
|
||||||
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const refreshVirtualList = () => {
|
const refreshVirtualList = () => {
|
||||||
@@ -497,6 +569,11 @@
|
|||||||
if (allObjects.length === 0 && streamFolders.length === 0 && !hasMoreObjects) {
|
if (allObjects.length === 0 && streamFolders.length === 0 && !hasMoreObjects) {
|
||||||
showEmptyState();
|
showEmptyState();
|
||||||
} else {
|
} else {
|
||||||
|
const isFiltering = currentFilterTerm && currentFilterTerm.length > 0;
|
||||||
|
const title = isFiltering ? 'No matches' : 'Empty folder';
|
||||||
|
const body = isFiltering
|
||||||
|
? `No objects match "${escapeHtml(currentFilterTerm)}".`
|
||||||
|
: `This folder contains no objects${hasMoreObjects ? ' yet. Loading more...' : '.'}`;
|
||||||
objectsTableBody.innerHTML = `
|
objectsTableBody.innerHTML = `
|
||||||
<tr>
|
<tr>
|
||||||
<td colspan="4" class="py-5">
|
<td colspan="4" class="py-5">
|
||||||
@@ -506,8 +583,8 @@
|
|||||||
<path d="M9.828 3h3.982a2 2 0 0 1 1.992 2.181l-.637 7A2 2 0 0 1 13.174 14H2.825a2 2 0 0 1-1.991-1.819l-.637-7a1.99 1.99 0 0 1 .342-1.31L.5 3a2 2 0 0 1 2-2h3.672a2 2 0 0 1 1.414.586l.828.828A2 2 0 0 0 9.828 3zm-8.322.12C1.72 3.042 1.95 3 2.19 3h5.396l-.707-.707A1 1 0 0 0 6.172 2H2.5a1 1 0 0 0-1 .981l.006.139z"/>
|
<path d="M9.828 3h3.982a2 2 0 0 1 1.992 2.181l-.637 7A2 2 0 0 1 13.174 14H2.825a2 2 0 0 1-1.991-1.819l-.637-7a1.99 1.99 0 0 1 .342-1.31L.5 3a2 2 0 0 1 2-2h3.672a2 2 0 0 1 1.414.586l.828.828A2 2 0 0 0 9.828 3zm-8.322.12C1.72 3.042 1.95 3 2.19 3h5.396l-.707-.707A1 1 0 0 0 6.172 2H2.5a1 1 0 0 0-1 .981l.006.139z"/>
|
||||||
</svg>
|
</svg>
|
||||||
</div>
|
</div>
|
||||||
<h6 class="mb-2">Empty folder</h6>
|
<h6 class="mb-2">${title}</h6>
|
||||||
<p class="text-muted small mb-0">This folder contains no objects${hasMoreObjects ? ' yet. Loading more...' : '.'}</p>
|
<p class="text-muted small mb-0">${body}</p>
|
||||||
</div>
|
</div>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
@@ -660,6 +737,10 @@
|
|||||||
break;
|
break;
|
||||||
case 'count':
|
case 'count':
|
||||||
totalObjectCount = msg.total_count || 0;
|
totalObjectCount = msg.total_count || 0;
|
||||||
|
if (!currentPrefix) {
|
||||||
|
bucketTotalObjects = totalObjectCount;
|
||||||
|
updateObjectCountBadge();
|
||||||
|
}
|
||||||
if (objectsLoadingRow) {
|
if (objectsLoadingRow) {
|
||||||
const loadingText = objectsLoadingRow.querySelector('p');
|
const loadingText = objectsLoadingRow.querySelector('p');
|
||||||
if (loadingText) loadingText.textContent = `Loading 0 of ${totalObjectCount.toLocaleString()} objects...`;
|
if (loadingText) loadingText.textContent = `Loading 0 of ${totalObjectCount.toLocaleString()} objects...`;
|
||||||
@@ -770,7 +851,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
totalObjectCount = data.total_count || 0;
|
totalObjectCount = data.total_count || 0;
|
||||||
if (!append && !currentPrefix && !useDelimiterMode) bucketTotalObjects = totalObjectCount;
|
if (!append && !currentPrefix) bucketTotalObjects = totalObjectCount;
|
||||||
nextContinuationToken = data.next_continuation_token;
|
nextContinuationToken = data.next_continuation_token;
|
||||||
|
|
||||||
if (!append && objectsLoadingRow) {
|
if (!append && objectsLoadingRow) {
|
||||||
@@ -907,12 +988,32 @@
|
|||||||
scrollContainer.addEventListener('scroll', handleVirtualScroll, { passive: true });
|
scrollContainer.addEventListener('scroll', handleVirtualScroll, { passive: true });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const isSentinelVisible = () => {
|
||||||
|
if (!scrollSentinel) return false;
|
||||||
|
const rect = scrollSentinel.getBoundingClientRect();
|
||||||
|
if (scrollContainer) {
|
||||||
|
const cr = scrollContainer.getBoundingClientRect();
|
||||||
|
return rect.top <= cr.bottom + 500 && rect.bottom >= cr.top - 500;
|
||||||
|
}
|
||||||
|
return rect.top <= window.innerHeight + 500 && rect.bottom >= -500;
|
||||||
|
};
|
||||||
|
|
||||||
|
const loadMoreOnSentinel = () => {
|
||||||
|
if (searchResults !== null) {
|
||||||
|
if (searchNextToken && !searchLoading) {
|
||||||
|
performServerSearch(currentFilterTerm, true);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (hasMoreObjects && !isLoadingObjects) {
|
||||||
|
loadObjects(true);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
if (scrollSentinel && scrollContainer) {
|
if (scrollSentinel && scrollContainer) {
|
||||||
const containerObserver = new IntersectionObserver((entries) => {
|
const containerObserver = new IntersectionObserver((entries) => {
|
||||||
entries.forEach(entry => {
|
entries.forEach(entry => {
|
||||||
if (entry.isIntersecting && hasMoreObjects && !isLoadingObjects) {
|
if (entry.isIntersecting) loadMoreOnSentinel();
|
||||||
loadObjects(true);
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
}, {
|
}, {
|
||||||
root: scrollContainer,
|
root: scrollContainer,
|
||||||
@@ -923,9 +1024,7 @@
|
|||||||
|
|
||||||
const viewportObserver = new IntersectionObserver((entries) => {
|
const viewportObserver = new IntersectionObserver((entries) => {
|
||||||
entries.forEach(entry => {
|
entries.forEach(entry => {
|
||||||
if (entry.isIntersecting && hasMoreObjects && !isLoadingObjects) {
|
if (entry.isIntersecting) loadMoreOnSentinel();
|
||||||
loadObjects(true);
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
}, {
|
}, {
|
||||||
root: null,
|
root: null,
|
||||||
@@ -1161,6 +1260,11 @@
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (folders.length === 0 && files.length === 0) {
|
if (folders.length === 0 && files.length === 0) {
|
||||||
|
const isFiltering = currentFilterTerm && currentFilterTerm.length > 0;
|
||||||
|
const title = isFiltering ? 'No matches' : 'Empty folder';
|
||||||
|
const body = isFiltering
|
||||||
|
? `No objects match "${escapeHtml(currentFilterTerm)}".`
|
||||||
|
: 'This folder contains no objects.';
|
||||||
const emptyRow = document.createElement('tr');
|
const emptyRow = document.createElement('tr');
|
||||||
emptyRow.innerHTML = `
|
emptyRow.innerHTML = `
|
||||||
<td colspan="4" class="py-5">
|
<td colspan="4" class="py-5">
|
||||||
@@ -1170,8 +1274,8 @@
|
|||||||
<path d="M9.828 3h3.982a2 2 0 0 1 1.992 2.181l-.637 7A2 2 0 0 1 13.174 14H2.825a2 2 0 0 1-1.991-1.819l-.637-7a1.99 1.99 0 0 1 .342-1.31L.5 3a2 2 0 0 1 2-2h3.672a2 2 0 0 1 1.414.586l.828.828A2 2 0 0 0 9.828 3zm-8.322.12C1.72 3.042 1.95 3 2.19 3h5.396l-.707-.707A1 1 0 0 0 6.172 2H2.5a1 1 0 0 0-1 .981l.006.139z"/>
|
<path d="M9.828 3h3.982a2 2 0 0 1 1.992 2.181l-.637 7A2 2 0 0 1 13.174 14H2.825a2 2 0 0 1-1.991-1.819l-.637-7a1.99 1.99 0 0 1 .342-1.31L.5 3a2 2 0 0 1 2-2h3.672a2 2 0 0 1 1.414.586l.828.828A2 2 0 0 0 9.828 3zm-8.322.12C1.72 3.042 1.95 3 2.19 3h5.396l-.707-.707A1 1 0 0 0 6.172 2H2.5a1 1 0 0 0-1 .981l.006.139z"/>
|
||||||
</svg>
|
</svg>
|
||||||
</div>
|
</div>
|
||||||
<h6 class="mb-2">Empty folder</h6>
|
<h6 class="mb-2">${title}</h6>
|
||||||
<p class="text-muted small mb-0">This folder contains no objects.</p>
|
<p class="text-muted small mb-0">${body}</p>
|
||||||
</div>
|
</div>
|
||||||
</td>
|
</td>
|
||||||
`;
|
`;
|
||||||
@@ -1351,6 +1455,11 @@
|
|||||||
});
|
});
|
||||||
|
|
||||||
const bulkActionsWrapper = document.getElementById('bulk-actions-wrapper');
|
const bulkActionsWrapper = document.getElementById('bulk-actions-wrapper');
|
||||||
|
const bulkDownloadButton = document.querySelector('[data-bulk-download-trigger]');
|
||||||
|
const updateBulkDownloadState = () => {
|
||||||
|
if (!bulkDownloadButton) return;
|
||||||
|
bulkDownloadButton.disabled = selectedRows.size === 0;
|
||||||
|
};
|
||||||
const updateBulkDeleteState = () => {
|
const updateBulkDeleteState = () => {
|
||||||
const selectedCount = selectedRows.size;
|
const selectedCount = selectedRows.size;
|
||||||
if (bulkDeleteButton) {
|
if (bulkDeleteButton) {
|
||||||
@@ -1377,6 +1486,7 @@
|
|||||||
selectAllCheckbox.checked = visibleSelectedCount > 0 && visibleSelectedCount === total && total > 0;
|
selectAllCheckbox.checked = visibleSelectedCount > 0 && visibleSelectedCount === total && total > 0;
|
||||||
selectAllCheckbox.indeterminate = visibleSelectedCount > 0 && visibleSelectedCount < total;
|
selectAllCheckbox.indeterminate = visibleSelectedCount > 0 && visibleSelectedCount < total;
|
||||||
}
|
}
|
||||||
|
updateBulkDownloadState();
|
||||||
};
|
};
|
||||||
|
|
||||||
function toggleRowSelection(row, shouldSelect) {
|
function toggleRowSelection(row, shouldSelect) {
|
||||||
@@ -1491,6 +1601,7 @@
|
|||||||
previewPanel.classList.add('d-none');
|
previewPanel.classList.add('d-none');
|
||||||
activeRow = null;
|
activeRow = null;
|
||||||
loadObjects(false);
|
loadObjects(false);
|
||||||
|
refreshBucketUsage();
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
bulkDeleteModal?.hide();
|
bulkDeleteModal?.hide();
|
||||||
showMessage({ title: 'Delete failed', body: (error && error.message) || 'Unable to delete selected objects', variant: 'danger' });
|
showMessage({ title: 'Delete failed', body: (error && error.message) || 'Unable to delete selected objects', variant: 'danger' });
|
||||||
@@ -1966,6 +2077,7 @@
|
|||||||
previewPanel.classList.add('d-none');
|
previewPanel.classList.add('d-none');
|
||||||
activeRow = null;
|
activeRow = null;
|
||||||
loadObjects(false);
|
loadObjects(false);
|
||||||
|
refreshBucketUsage();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (deleteModal) deleteModal.hide();
|
if (deleteModal) deleteModal.hide();
|
||||||
showMessage({ title: 'Delete failed', body: err.message || 'Unable to delete object', variant: 'danger' });
|
showMessage({ title: 'Delete failed', body: err.message || 'Unable to delete object', variant: 'danger' });
|
||||||
@@ -2202,47 +2314,69 @@
|
|||||||
const filterWarningText = document.getElementById('filter-warning-text');
|
const filterWarningText = document.getElementById('filter-warning-text');
|
||||||
const folderViewStatus = document.getElementById('folder-view-status');
|
const folderViewStatus = document.getElementById('folder-view-status');
|
||||||
|
|
||||||
const updateFilterWarning = () => {
|
|
||||||
if (!filterWarning) return;
|
|
||||||
const isFiltering = currentFilterTerm.length > 0;
|
|
||||||
if (isFiltering && hasMoreObjects) {
|
|
||||||
filterWarning.classList.remove('d-none');
|
|
||||||
} else {
|
|
||||||
filterWarning.classList.add('d-none');
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let searchDebounceTimer = null;
|
let searchDebounceTimer = null;
|
||||||
let searchAbortController = null;
|
let searchAbortController = null;
|
||||||
let searchResults = null;
|
let searchResults = null;
|
||||||
|
let searchNextToken = null;
|
||||||
|
let searchLoading = false;
|
||||||
|
const SEARCH_PAGE_SIZE = 500;
|
||||||
|
|
||||||
const performServerSearch = async (term) => {
|
const updateFilterWarning = () => {
|
||||||
if (searchAbortController) searchAbortController.abort();
|
if (!filterWarning) return;
|
||||||
searchAbortController = new AbortController();
|
filterWarning.classList.add('d-none');
|
||||||
|
};
|
||||||
|
|
||||||
|
const performServerSearch = async (term, append = false) => {
|
||||||
|
if (!append && searchAbortController) searchAbortController.abort();
|
||||||
|
if (append && (searchLoading || !searchNextToken)) return;
|
||||||
|
if (!append) {
|
||||||
|
searchAbortController = new AbortController();
|
||||||
|
}
|
||||||
|
searchLoading = true;
|
||||||
|
if (append && loadMoreSpinner) loadMoreSpinner.classList.remove('d-none');
|
||||||
|
|
||||||
|
let succeeded = false;
|
||||||
try {
|
try {
|
||||||
const params = new URLSearchParams({ q: term, limit: '500' });
|
const params = new URLSearchParams({ q: term, limit: String(SEARCH_PAGE_SIZE) });
|
||||||
if (currentPrefix) params.set('prefix', currentPrefix);
|
if (currentPrefix) params.set('prefix', currentPrefix);
|
||||||
|
if (append && searchNextToken) params.set('start_after', searchNextToken);
|
||||||
const searchUrl = objectsStreamUrl.replace('/stream', '/search');
|
const searchUrl = objectsStreamUrl.replace('/stream', '/search');
|
||||||
const response = await fetch(`${searchUrl}?${params}`, {
|
const response = await fetch(`${searchUrl}?${params}`, {
|
||||||
signal: searchAbortController.signal
|
signal: searchAbortController?.signal
|
||||||
});
|
});
|
||||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
searchResults = (data.results || []).map(obj => processStreamObject(obj));
|
const newResults = (data.results || []).map(obj => processStreamObject(obj));
|
||||||
|
if (append && Array.isArray(searchResults)) {
|
||||||
|
searchResults = searchResults.concat(newResults);
|
||||||
|
} else {
|
||||||
|
searchResults = newResults;
|
||||||
|
}
|
||||||
|
searchNextToken = data.truncated ? (data.next_token || null) : null;
|
||||||
memoizedVisibleItems = null;
|
memoizedVisibleItems = null;
|
||||||
memoizedInputs = { objectCount: -1, folderCount: -1, prefix: null, filterTerm: null };
|
memoizedInputs = { objectCount: -1, folderCount: -1, prefix: null, filterTerm: null };
|
||||||
refreshVirtualList();
|
refreshVirtualList();
|
||||||
if (loadMoreStatus) {
|
if (loadMoreStatus) {
|
||||||
const countText = searchResults.length.toLocaleString();
|
const countText = searchResults.length.toLocaleString();
|
||||||
const truncated = data.truncated ? '+' : '';
|
const more = searchNextToken ? '+' : '';
|
||||||
loadMoreStatus.textContent = `${countText}${truncated} result${searchResults.length !== 1 ? 's' : ''}`;
|
const noun = searchResults.length === 1 ? 'result' : 'results';
|
||||||
|
loadMoreStatus.textContent = searchNextToken
|
||||||
|
? `${countText}${more} ${noun} (scroll to load more)`
|
||||||
|
: `${countText} ${noun}`;
|
||||||
}
|
}
|
||||||
|
succeeded = true;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (e.name === 'AbortError') return;
|
if (e.name === 'AbortError') return;
|
||||||
if (loadMoreStatus) {
|
if (loadMoreStatus) {
|
||||||
loadMoreStatus.textContent = 'Search failed';
|
loadMoreStatus.textContent = 'Search failed (scroll to retry)';
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
|
searchLoading = false;
|
||||||
|
if (loadMoreSpinner) loadMoreSpinner.classList.add('d-none');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (succeeded && searchNextToken && !searchLoading && isSentinelVisible()) {
|
||||||
|
performServerSearch(currentFilterTerm, true);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -2262,6 +2396,7 @@
|
|||||||
if (!isFiltering && wasFiltering) {
|
if (!isFiltering && wasFiltering) {
|
||||||
if (searchAbortController) searchAbortController.abort();
|
if (searchAbortController) searchAbortController.abort();
|
||||||
searchResults = null;
|
searchResults = null;
|
||||||
|
searchNextToken = null;
|
||||||
memoizedVisibleItems = null;
|
memoizedVisibleItems = null;
|
||||||
memoizedInputs = { objectCount: -1, folderCount: -1, prefix: null, filterTerm: null };
|
memoizedInputs = { objectCount: -1, folderCount: -1, prefix: null, filterTerm: null };
|
||||||
if (loadMoreStatus) {
|
if (loadMoreStatus) {
|
||||||
@@ -3071,6 +3206,7 @@
|
|||||||
} else if (errorCount > 0) {
|
} else if (errorCount > 0) {
|
||||||
showMessage({ title: 'Upload failed', body: `${errorCount} file(s) failed to upload.`, variant: 'danger' });
|
showMessage({ title: 'Upload failed', body: `${errorCount} file(s) failed to upload.`, variant: 'danger' });
|
||||||
}
|
}
|
||||||
|
if (successCount > 0) refreshBucketUsage();
|
||||||
};
|
};
|
||||||
|
|
||||||
const performBulkUpload = async (files) => {
|
const performBulkUpload = async (files) => {
|
||||||
@@ -3238,15 +3374,8 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const bulkDownloadButton = document.querySelector('[data-bulk-download-trigger]');
|
|
||||||
const bulkDownloadEndpoint = document.getElementById('objects-drop-zone')?.dataset.bulkDownloadEndpoint;
|
const bulkDownloadEndpoint = document.getElementById('objects-drop-zone')?.dataset.bulkDownloadEndpoint;
|
||||||
|
|
||||||
const updateBulkDownloadState = () => {
|
|
||||||
if (!bulkDownloadButton) return;
|
|
||||||
const selectedCount = document.querySelectorAll('[data-object-select]:checked').length;
|
|
||||||
bulkDownloadButton.disabled = selectedCount === 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
selectAllCheckbox?.addEventListener('change', (event) => {
|
selectAllCheckbox?.addEventListener('change', (event) => {
|
||||||
const shouldSelect = Boolean(event.target?.checked);
|
const shouldSelect = Boolean(event.target?.checked);
|
||||||
|
|
||||||
@@ -3281,7 +3410,6 @@
|
|||||||
});
|
});
|
||||||
|
|
||||||
updateBulkDeleteState();
|
updateBulkDeleteState();
|
||||||
setTimeout(updateBulkDownloadState, 0);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
bulkDownloadButton?.addEventListener('click', async () => {
|
bulkDownloadButton?.addEventListener('click', async () => {
|
||||||
@@ -4329,10 +4457,25 @@
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (lifecycleHistoryCard) {
|
if (lifecycleHistoryCard) {
|
||||||
loadLifecycleHistory();
|
const lifecycleTab = document.getElementById('lifecycle-tab');
|
||||||
if (window.pollingManager) {
|
const lifecyclePane = document.getElementById('lifecycle-pane');
|
||||||
window.pollingManager.start('lifecycle', loadLifecycleHistory);
|
const startLifecyclePolling = () => {
|
||||||
|
if (window.pollingManager) {
|
||||||
|
window.pollingManager.start('lifecycle', loadLifecycleHistory);
|
||||||
|
} else {
|
||||||
|
loadLifecycleHistory();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const stopLifecyclePolling = () => {
|
||||||
|
if (window.pollingManager) {
|
||||||
|
window.pollingManager.stop('lifecycle');
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if (lifecyclePane && lifecyclePane.classList.contains('show') && lifecyclePane.classList.contains('active')) {
|
||||||
|
startLifecyclePolling();
|
||||||
}
|
}
|
||||||
|
lifecycleTab?.addEventListener('shown.bs.tab', startLifecyclePolling);
|
||||||
|
lifecycleTab?.addEventListener('hidden.bs.tab', stopLifecyclePolling);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (corsCard) loadCorsRules();
|
if (corsCard) loadCorsRules();
|
||||||
@@ -4542,6 +4685,16 @@
|
|||||||
var maxObjInput = document.getElementById('max_objects');
|
var maxObjInput = document.getElementById('max_objects');
|
||||||
if (maxMbInput) maxMbInput.value = maxBytes ? Math.floor(maxBytes / 1048576) : '';
|
if (maxMbInput) maxMbInput.value = maxBytes ? Math.floor(maxBytes / 1048576) : '';
|
||||||
if (maxObjInput) maxObjInput.value = maxObjects || '';
|
if (maxObjInput) maxObjInput.value = maxObjects || '';
|
||||||
|
|
||||||
|
var objectsCard = document.querySelector('[data-usage-objects]');
|
||||||
|
if (objectsCard) {
|
||||||
|
objectsCard.dataset.maxObjects = maxObjects && maxObjects > 0 ? String(maxObjects) : '';
|
||||||
|
}
|
||||||
|
var bytesCard = document.querySelector('[data-usage-bytes]');
|
||||||
|
if (bytesCard) {
|
||||||
|
bytesCard.dataset.maxBytes = maxBytes && maxBytes > 0 ? String(maxBytes) : '';
|
||||||
|
}
|
||||||
|
redrawUsageLimits();
|
||||||
}
|
}
|
||||||
|
|
||||||
function updatePolicyCard(hasPolicy, preset) {
|
function updatePolicyCard(hasPolicy, preset) {
|
||||||
@@ -4815,7 +4968,7 @@
|
|||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
window.UICore.submitFormAjax(deleteBucketForm, {
|
window.UICore.submitFormAjax(deleteBucketForm, {
|
||||||
onSuccess: function () {
|
onSuccess: function () {
|
||||||
sessionStorage.setItem('flashMessage', JSON.stringify({ title: 'Bucket deleted', variant: 'success' }));
|
sessionStorage.setItem('flashMessage', JSON.stringify({ title: 'Success', body: 'Bucket deleted', variant: 'success' }));
|
||||||
window.location.href = window.BucketDetailConfig?.endpoints?.bucketsOverview || '/ui/buckets';
|
window.location.href = window.BucketDetailConfig?.endpoints?.bucketsOverview || '/ui/buckets';
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ window.IAMManagement = (function() {
|
|||||||
];
|
];
|
||||||
|
|
||||||
var policyTemplates = {
|
var policyTemplates = {
|
||||||
full: [{ bucket: '*', actions: ['list', 'read', 'write', 'delete', 'share', 'policy', 'create_bucket', 'delete_bucket', 'replication', 'lifecycle', 'cors', 'versioning', 'tagging', 'encryption', 'quota', 'object_lock', 'notification', 'logging', 'website', 'iam:*'] }],
|
full: [{ bucket: '*', actions: ['*'] }],
|
||||||
readonly: [{ bucket: '*', actions: ['list', 'read'] }],
|
readonly: [{ bucket: '*', actions: ['list', 'read'] }],
|
||||||
writer: [{ bucket: '*', actions: ['list', 'read', 'write'] }],
|
writer: [{ bucket: '*', actions: ['list', 'read', 'write'] }],
|
||||||
operator: [{ bucket: '*', actions: ['list', 'read', 'write', 'delete', 'create_bucket', 'delete_bucket'] }],
|
operator: [{ bucket: '*', actions: ['list', 'read', 'write', 'delete', 'create_bucket', 'delete_bucket'] }],
|
||||||
@@ -39,7 +39,7 @@ window.IAMManagement = (function() {
|
|||||||
function isAdminUser(policies) {
|
function isAdminUser(policies) {
|
||||||
if (!policies || !policies.length) return false;
|
if (!policies || !policies.length) return false;
|
||||||
return policies.some(function(p) {
|
return policies.some(function(p) {
|
||||||
return p.actions && (p.actions.indexOf('iam:*') >= 0 || p.actions.indexOf('*') >= 0);
|
return p.bucket === '*' && p.actions && p.actions.indexOf('*') >= 0;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,292 +0,0 @@
|
|||||||
import os
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
|
|
||||||
TEMPLATE_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
||||||
|
|
||||||
TERNARY_RE = re.compile(
|
|
||||||
r"""(\{\{\s*)
|
|
||||||
(?:"([^"]*)"|'([^']*)') # literal A
|
|
||||||
\s+if\s+
|
|
||||||
([^{}]+?) # condition
|
|
||||||
\s+else\s+
|
|
||||||
(?:"([^"]*)"|'([^']*)') # literal B
|
|
||||||
(\s*\}\})""",
|
|
||||||
re.VERBOSE,
|
|
||||||
)
|
|
||||||
|
|
||||||
TERNARY_SET_RE = re.compile(
|
|
||||||
r"""(\{%\s*set\s+([A-Za-z_][A-Za-z_0-9]*)\s*=\s*)
|
|
||||||
(?:"([^"]*)"|'([^']*)')
|
|
||||||
\s+if\s+
|
|
||||||
([^{}]+?)
|
|
||||||
\s+else\s+
|
|
||||||
(?:"([^"]*)"|'([^']*)')
|
|
||||||
(\s*%\})""",
|
|
||||||
re.VERBOSE,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def convert_single_quoted_strings_in_expressions(text: str) -> str:
|
|
||||||
"""Inside {{...}} or {%...%}, swap ' for " around tokens that look like strings."""
|
|
||||||
def fix(m):
|
|
||||||
body = m.group(2)
|
|
||||||
body_fixed = re.sub(r"'([^'\\\n]*)'", r'"\1"', body)
|
|
||||||
return m.group(1) + body_fixed + m.group(3)
|
|
||||||
|
|
||||||
return re.sub(
|
|
||||||
r"(\{[{%])([^{}]*?)([}%]\})",
|
|
||||||
fix,
|
|
||||||
text,
|
|
||||||
flags=re.DOTALL,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def convert_inline_ternary(text: str) -> str:
|
|
||||||
def repl_expr(m):
|
|
||||||
a = m.group(2) if m.group(2) is not None else m.group(3)
|
|
||||||
cond = m.group(4)
|
|
||||||
b = m.group(5) if m.group(5) is not None else m.group(6)
|
|
||||||
return (
|
|
||||||
'{% if ' + cond + ' %}' + a + '{% else %}' + b + '{% endif %}'
|
|
||||||
)
|
|
||||||
|
|
||||||
def repl_set(m):
|
|
||||||
varname = m.group(2)
|
|
||||||
a = m.group(3) if m.group(3) is not None else m.group(4)
|
|
||||||
cond = m.group(5)
|
|
||||||
b = m.group(6) if m.group(6) is not None else m.group(7)
|
|
||||||
return (
|
|
||||||
'{% if ' + cond + ' %}{% set ' + varname + ' = "' + a + '" %}'
|
|
||||||
'{% else %}{% set ' + varname + ' = "' + b + '" %}{% endif %}'
|
|
||||||
)
|
|
||||||
|
|
||||||
prev = None
|
|
||||||
while prev != text:
|
|
||||||
prev = text
|
|
||||||
text = TERNARY_SET_RE.sub(repl_set, text)
|
|
||||||
text = TERNARY_RE.sub(repl_expr, text)
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def convert_request_args(text: str) -> str:
|
|
||||||
text = re.sub(
|
|
||||||
r'request\.args\.get\(\s*"([^"]+)"\s*,\s*"([^"]*)"\s*\)',
|
|
||||||
r'request_args.\1 | default(value="\2")',
|
|
||||||
text,
|
|
||||||
)
|
|
||||||
text = re.sub(
|
|
||||||
r'request\.args\.get\(\s*"([^"]+)"\s*\)',
|
|
||||||
r'request_args.\1',
|
|
||||||
text,
|
|
||||||
)
|
|
||||||
text = text.replace('request.endpoint', 'current_endpoint')
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def convert_items_keys(text: str) -> str:
|
|
||||||
text = re.sub(r'\.items\(\)', '', text)
|
|
||||||
text = re.sub(r'\.keys\(\)', '', text)
|
|
||||||
text = re.sub(r'\.values\(\)', '', text)
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def convert_tojson(text: str) -> str:
|
|
||||||
text = re.sub(r'\|\s*tojson\b', '| json_encode | safe', text)
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def convert_is_none(text: str) -> str:
|
|
||||||
text = re.sub(r'\bis\s+not\s+none\b', '!= null', text)
|
|
||||||
text = re.sub(r'\bis\s+none\b', '== null', text)
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def convert_namespace(text: str) -> str:
|
|
||||||
def repl(m):
|
|
||||||
body = m.group(1)
|
|
||||||
assigns = [a.strip() for a in body.split(',')]
|
|
||||||
return '{# namespace shim #}'
|
|
||||||
|
|
||||||
text = re.sub(
|
|
||||||
r'\{%\s*set\s+ns\s*=\s*namespace\(([^)]*)\)\s*%\}',
|
|
||||||
repl,
|
|
||||||
text,
|
|
||||||
)
|
|
||||||
text = re.sub(r'\bns\.([A-Za-z_][A-Za-z_0-9]*)\s*=\s*', r'{% set_global \1 = ', text)
|
|
||||||
text = re.sub(r'\bns\.([A-Za-z_][A-Za-z_0-9]*)', r'\1', text)
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def convert_url_for_positional(text: str) -> str:
|
|
||||||
"""url_for("x", ...) -> url_for(endpoint="x", ...)"""
|
|
||||||
def repl(m):
|
|
||||||
prefix = m.group(1)
|
|
||||||
endpoint = m.group(2)
|
|
||||||
rest = m.group(3) or ''
|
|
||||||
rest = rest.strip()
|
|
||||||
if rest.startswith(','):
|
|
||||||
rest = rest[1:].strip()
|
|
||||||
if rest:
|
|
||||||
return f'{prefix}(endpoint="{endpoint}", {rest})'
|
|
||||||
return f'{prefix}(endpoint="{endpoint}")'
|
|
||||||
|
|
||||||
pattern = re.compile(r'(url_for)\(\s*"([^"]+)"\s*((?:,[^()]*)?)\)')
|
|
||||||
prev = None
|
|
||||||
while prev != text:
|
|
||||||
prev = text
|
|
||||||
text = pattern.sub(repl, text)
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def convert_d_filter(text: str) -> str:
|
|
||||||
text = re.sub(r'\|\s*d\(\s*([^)]*?)\s*\)', lambda m: f'| default(value={m.group(1) or 0})', text)
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def convert_replace_filter(text: str) -> str:
|
|
||||||
def repl(m):
|
|
||||||
a = m.group(1)
|
|
||||||
b = m.group(2)
|
|
||||||
return f'| replace(from="{a}", to="{b}")'
|
|
||||||
text = re.sub(r'\|\s*replace\(\s*"([^"]*)"\s*,\s*"([^"]*)"\s*\)', repl, text)
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def convert_truncate_filter(text: str) -> str:
|
|
||||||
def repl(m):
|
|
||||||
n = m.group(1)
|
|
||||||
return f'| truncate(length={n})'
|
|
||||||
text = re.sub(r'\|\s*truncate\(\s*(\d+)\s*(?:,[^)]*)?\)', repl, text)
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def convert_strip_method(text: str) -> str:
|
|
||||||
text = re.sub(r'(\b[A-Za-z_][A-Za-z_0-9.\[\]"]*)\s*\.\s*strip\(\s*\)', r'\1 | trim', text)
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def convert_split_method(text: str) -> str:
|
|
||||||
def repl(m):
|
|
||||||
obj = m.group(1)
|
|
||||||
sep = m.group(2)
|
|
||||||
return f'{obj} | split(pat="{sep}")'
|
|
||||||
text = re.sub(r'(\b[A-Za-z_][A-Za-z_0-9.]*)\s*\.\s*split\(\s*"([^"]*)"\s*\)', repl, text)
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def convert_python_slice(text: str) -> str:
|
|
||||||
def repl_colon(m):
|
|
||||||
obj = m.group(1)
|
|
||||||
start = m.group(2) or '0'
|
|
||||||
end = m.group(3)
|
|
||||||
if start.startswith('-') or (end and end.startswith('-')):
|
|
||||||
return m.group(0)
|
|
||||||
if end:
|
|
||||||
return f'{obj} | slice(start={start}, end={end})'
|
|
||||||
return f'{obj} | slice(start={start})'
|
|
||||||
|
|
||||||
def repl_neg_end(m):
|
|
||||||
obj = m.group(1)
|
|
||||||
n = m.group(2)
|
|
||||||
return f'{obj} | slice(start=-{n})'
|
|
||||||
|
|
||||||
text = re.sub(
|
|
||||||
r'(\b[A-Za-z_][A-Za-z_0-9.]*)\[\s*(-?\d*)\s*:\s*(-?\d*)\s*\]',
|
|
||||||
repl_colon,
|
|
||||||
text,
|
|
||||||
)
|
|
||||||
text = re.sub(
|
|
||||||
r'(\b[A-Za-z_][A-Za-z_0-9.]*)\|\s*slice\(start=-(\d+)\s*,\s*end=\s*\)',
|
|
||||||
repl_neg_end,
|
|
||||||
text,
|
|
||||||
)
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def convert_inline_ternary_expr(text: str) -> str:
|
|
||||||
"""Handle arbitrary ternary inside {{ ... }}: A if COND else B -> {% if COND %}A{% else %}B{% endif %}"""
|
|
||||||
out_lines = []
|
|
||||||
for line in text.split('\n'):
|
|
||||||
out_lines.append(_convert_line_ternary(line))
|
|
||||||
return '\n'.join(out_lines)
|
|
||||||
|
|
||||||
|
|
||||||
def _convert_line_ternary(line: str) -> str:
|
|
||||||
if '{{' not in line or ' if ' not in line or ' else ' not in line:
|
|
||||||
return line
|
|
||||||
prev = None
|
|
||||||
while prev != line:
|
|
||||||
prev = line
|
|
||||||
m = re.search(r'\{\{\s*([^{}]+?)\s+if\s+([^{}]+?)\s+else\s+([^{}]+?)\s*\}\}', line)
|
|
||||||
if not m:
|
|
||||||
break
|
|
||||||
replacement = '{% if ' + m.group(2) + ' %}{{ ' + m.group(1) + ' }}{% else %}{{ ' + m.group(3) + ' }}{% endif %}'
|
|
||||||
line = line[:m.start()] + replacement + line[m.end():]
|
|
||||||
return line
|
|
||||||
|
|
||||||
|
|
||||||
def convert_dict_get(text: str) -> str:
|
|
||||||
"""Convert X.get("key", default) -> X.key | default(value=default) when simple."""
|
|
||||||
pattern = re.compile(
|
|
||||||
r'([A-Za-z_][A-Za-z_0-9]*(?:\.[A-Za-z_][A-Za-z_0-9]*)*)'
|
|
||||||
r'\.get\(\s*"([A-Za-z_][A-Za-z_0-9]*)"\s*(?:,\s*([^(){}]+?))?\s*\)'
|
|
||||||
)
|
|
||||||
|
|
||||||
def repl(m):
|
|
||||||
obj = m.group(1)
|
|
||||||
key = m.group(2)
|
|
||||||
default = (m.group(3) or '').strip()
|
|
||||||
if default:
|
|
||||||
return f'{obj}.{key} | default(value={default})'
|
|
||||||
return f'{obj}.{key}'
|
|
||||||
|
|
||||||
prev = None
|
|
||||||
while prev != text:
|
|
||||||
prev = text
|
|
||||||
text = pattern.sub(repl, text)
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def convert_file(path: str) -> bool:
|
|
||||||
with open(path, 'r', encoding='utf-8') as f:
|
|
||||||
original = f.read()
|
|
||||||
text = original
|
|
||||||
text = convert_single_quoted_strings_in_expressions(text)
|
|
||||||
text = convert_inline_ternary(text)
|
|
||||||
text = convert_request_args(text)
|
|
||||||
text = convert_items_keys(text)
|
|
||||||
text = convert_tojson(text)
|
|
||||||
text = convert_is_none(text)
|
|
||||||
text = convert_namespace(text)
|
|
||||||
text = convert_dict_get(text)
|
|
||||||
text = convert_url_for_positional(text)
|
|
||||||
text = convert_d_filter(text)
|
|
||||||
text = convert_replace_filter(text)
|
|
||||||
text = convert_truncate_filter(text)
|
|
||||||
text = convert_strip_method(text)
|
|
||||||
text = convert_split_method(text)
|
|
||||||
text = convert_python_slice(text)
|
|
||||||
text = convert_inline_ternary_expr(text)
|
|
||||||
if text != original:
|
|
||||||
with open(path, 'w', encoding='utf-8', newline='\n') as f:
|
|
||||||
f.write(text)
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
changed = []
|
|
||||||
for name in sorted(os.listdir(TEMPLATE_DIR)):
|
|
||||||
if not name.endswith('.html'):
|
|
||||||
continue
|
|
||||||
p = os.path.join(TEMPLATE_DIR, name)
|
|
||||||
if convert_file(p):
|
|
||||||
changed.append(name)
|
|
||||||
print('Changed:', len(changed))
|
|
||||||
for c in changed:
|
|
||||||
print(' -', c)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@@ -87,19 +87,18 @@
|
|||||||
</svg>
|
</svg>
|
||||||
<span>Connections</span>
|
<span>Connections</span>
|
||||||
</a>
|
</a>
|
||||||
<a href="{{ url_for(endpoint="ui.metrics_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.metrics_dashboard" %}active{% endif %}">
|
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
|
||||||
<path d="M8 4a.5.5 0 0 1 .5.5V6a.5.5 0 0 1-1 0V4.5A.5.5 0 0 1 8 4zM3.732 5.732a.5.5 0 0 1 .707 0l.915.914a.5.5 0 1 1-.708.708l-.914-.915a.5.5 0 0 1 0-.707zM2 10a.5.5 0 0 1 .5-.5h1.586a.5.5 0 0 1 0 1H2.5A.5.5 0 0 1 2 10zm9.5 0a.5.5 0 0 1 .5-.5h1.5a.5.5 0 0 1 0 1H12a.5.5 0 0 1-.5-.5zm.754-4.246a.389.389 0 0 0-.527-.02L7.547 9.31a.91.91 0 1 0 1.302 1.258l3.434-4.297a.389.389 0 0 0-.029-.518z"/>
|
|
||||||
<path fill-rule="evenodd" d="M0 10a8 8 0 1 1 15.547 2.661c-.442 1.253-1.845 1.602-2.932 1.25C11.309 13.488 9.475 13 8 13c-1.474 0-3.31.488-4.615.911-1.087.352-2.49.003-2.932-1.25A7.988 7.988 0 0 1 0 10zm8-7a7 7 0 0 0-6.603 9.329c.203.575.923.876 1.68.63C4.397 12.533 6.358 12 8 12s3.604.532 4.923.96c.757.245 1.477-.056 1.68-.631A7 7 0 0 0 8 3z"/>
|
|
||||||
</svg>
|
|
||||||
<span>Metrics</span>
|
|
||||||
</a>
|
|
||||||
<a href="{{ url_for(endpoint="ui.sites_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.sites_dashboard" %}active{% endif %}">
|
<a href="{{ url_for(endpoint="ui.sites_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.sites_dashboard" %}active{% endif %}">
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||||
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472zM3.82 11a13.652 13.652 0 0 1-.312-2.5h-2.49c.062.89.291 1.733.656 2.5H3.82zm6.853 3.472A7.024 7.024 0 0 0 13.745 12H11.91a9.27 9.27 0 0 1-.64 1.539 6.688 6.688 0 0 1-.597.933zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855.173-.324.33-.682.468-1.068H8.5zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.65 13.65 0 0 1-.312 2.5zm2.802-3.5a6.959 6.959 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5h2.49zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7.024 7.024 0 0 0-3.072-2.472c.218.284.418.598.597.933zM10.855 4a7.966 7.966 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4h2.355z"/>
|
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472zM3.82 11a13.652 13.652 0 0 1-.312-2.5h-2.49c.062.89.291 1.733.656 2.5H3.82zm6.853 3.472A7.024 7.024 0 0 0 13.745 12H11.91a9.27 9.27 0 0 1-.64 1.539 6.688 6.688 0 0 1-.597.933zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855.173-.324.33-.682.468-1.068H8.5zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.65 13.65 0 0 1-.312 2.5zm2.802-3.5a6.959 6.959 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5h2.49zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7.024 7.024 0 0 0-3.072-2.472c.218.284.418.598.597.933zM10.855 4a7.966 7.966 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4h2.355z"/>
|
||||||
</svg>
|
</svg>
|
||||||
<span>Sites</span>
|
<span>Sites</span>
|
||||||
</a>
|
</a>
|
||||||
|
<a href="{{ url_for(endpoint="ui.cluster_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.cluster_dashboard" %}active{% endif %}">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||||
|
<path d="M7.752.066a.5.5 0 0 1 .496 0l3.75 2.143a.5.5 0 0 1 .252.434v3.995l3.498 2A.5.5 0 0 1 16 9.07v4.286a.5.5 0 0 1-.252.434l-3.75 2.143a.5.5 0 0 1-.496 0l-3.502-2-3.502 2.001a.5.5 0 0 1-.496 0l-3.75-2.143A.5.5 0 0 1 0 13.357V9.071a.5.5 0 0 1 .252-.434L3.75 6.638V2.643a.5.5 0 0 1 .252-.434L7.752.066ZM4.25 7.504 1.508 9.071l2.742 1.567 2.742-1.567L4.25 7.504ZM7.5 9.933l-2.75 1.571v3.134l2.75-1.571V9.933Zm1 3.134 2.75 1.571v-3.134L8.5 9.933v3.134Zm.508-3.996 2.742 1.567 2.742-1.567-2.742-1.567-2.742 1.567Zm2.242-2.433V3.504L8.5 5.076V8.21l2.75-1.572ZM7.5 8.21V5.076L4.75 3.504v3.134L7.5 8.21ZM5.258 2.643 8 4.21l2.742-1.567L8 1.076 5.258 2.643ZM15 9.933l-2.75 1.571v3.134L15 13.067V9.933ZM3.75 14.638v-3.134L1 9.933v3.134l2.75 1.571Z"/>
|
||||||
|
</svg>
|
||||||
|
<span>Cluster</span>
|
||||||
|
</a>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% if website_hosting_nav %}
|
{% if website_hosting_nav %}
|
||||||
<a href="{{ url_for(endpoint="ui.website_domains_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.website_domains_dashboard" %}active{% endif %}">
|
<a href="{{ url_for(endpoint="ui.website_domains_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.website_domains_dashboard" %}active{% endif %}">
|
||||||
@@ -111,6 +110,13 @@
|
|||||||
</a>
|
</a>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% if can_manage_iam %}
|
{% if can_manage_iam %}
|
||||||
|
<a href="{{ url_for(endpoint="ui.metrics_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.metrics_dashboard" %}active{% endif %}">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||||
|
<path d="M8 4a.5.5 0 0 1 .5.5V6a.5.5 0 0 1-1 0V4.5A.5.5 0 0 1 8 4zM3.732 5.732a.5.5 0 0 1 .707 0l.915.914a.5.5 0 1 1-.708.708l-.914-.915a.5.5 0 0 1 0-.707zM2 10a.5.5 0 0 1 .5-.5h1.586a.5.5 0 0 1 0 1H2.5A.5.5 0 0 1 2 10zm9.5 0a.5.5 0 0 1 .5-.5h1.5a.5.5 0 0 1 0 1H12a.5.5 0 0 1-.5-.5zm.754-4.246a.389.389 0 0 0-.527-.02L7.547 9.31a.91.91 0 1 0 1.302 1.258l3.434-4.297a.389.389 0 0 0-.029-.518z"/>
|
||||||
|
<path fill-rule="evenodd" d="M0 10a8 8 0 1 1 15.547 2.661c-.442 1.253-1.845 1.602-2.932 1.25C11.309 13.488 9.475 13 8 13c-1.474 0-3.31.488-4.615.911-1.087.352-2.49.003-2.932-1.25A7.988 7.988 0 0 1 0 10zm8-7a7 7 0 0 0-6.603 9.329c.203.575.923.876 1.68.63C4.397 12.533 6.358 12 8 12s3.604.532 4.923.96c.757.245 1.477-.056 1.68-.631A7 7 0 0 0 8 3z"/>
|
||||||
|
</svg>
|
||||||
|
<span>Metrics</span>
|
||||||
|
</a>
|
||||||
<a href="{{ url_for(endpoint="ui.system_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.system_dashboard" %}active{% endif %}">
|
<a href="{{ url_for(endpoint="ui.system_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.system_dashboard" %}active{% endif %}">
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||||
<path d="M9.405 1.05c-.413-1.4-2.397-1.4-2.81 0l-.1.34a1.464 1.464 0 0 1-2.105.872l-.31-.17c-1.283-.698-2.686.705-1.987 1.987l.169.311c.446.82.023 1.841-.872 2.105l-.34.1c-1.4.413-1.4 2.397 0 2.81l.34.1a1.464 1.464 0 0 1 .872 2.105l-.17.31c-.698 1.283.705 2.686 1.987 1.987l.311-.169a1.464 1.464 0 0 1 2.105.872l.1.34c.413 1.4 2.397 1.4 2.81 0l.1-.34a1.464 1.464 0 0 1 2.105-.872l.31.17c1.283.698 2.686-.705 1.987-1.987l-.169-.311a1.464 1.464 0 0 1 .872-2.105l.34-.1c1.4-.413 1.4-2.397 0-2.81l-.34-.1a1.464 1.464 0 0 1-.872-2.105l.17-.31c.698-1.283-.705-2.686-1.987-1.987l-.311.169a1.464 1.464 0 0 1-2.105-.872l-.1-.34zM8 10.93a2.929 2.929 0 1 1 0-5.86 2.929 2.929 0 0 1 0 5.858z"/>
|
<path d="M9.405 1.05c-.413-1.4-2.397-1.4-2.81 0l-.1.34a1.464 1.464 0 0 1-2.105.872l-.31-.17c-1.283-.698-2.686.705-1.987 1.987l.169.311c.446.82.023 1.841-.872 2.105l-.34.1c-1.4.413-1.4 2.397 0 2.81l.34.1a1.464 1.464 0 0 1 .872 2.105l-.17.31c-.698 1.283.705 2.686 1.987 1.987l.311-.169a1.464 1.464 0 0 1 2.105.872l.1.34c.413 1.4 2.397 1.4 2.81 0l.1-.34a1.464 1.464 0 0 1 2.105-.872l.31.17c1.283.698 2.686-.705 1.987-1.987l-.169-.311a1.464 1.464 0 0 1 .872-2.105l.34-.1c1.4-.413 1.4-2.397 0-2.81l-.34-.1a1.464 1.464 0 0 1-.872-2.105l.17-.31c.698-1.283-.705-2.686-1.987-1.987l-.311.169a1.464 1.464 0 0 1-2.105-.872l-.1-.34zM8 10.93a2.929 2.929 0 1 1 0-5.86 2.929 2.929 0 0 1 0 5.858z"/>
|
||||||
@@ -195,19 +201,18 @@
|
|||||||
</svg>
|
</svg>
|
||||||
<span class="sidebar-link-text">Connections</span>
|
<span class="sidebar-link-text">Connections</span>
|
||||||
</a>
|
</a>
|
||||||
<a href="{{ url_for(endpoint="ui.metrics_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.metrics_dashboard" %}active{% endif %}" data-tooltip="Metrics">
|
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
|
||||||
<path d="M8 4a.5.5 0 0 1 .5.5V6a.5.5 0 0 1-1 0V4.5A.5.5 0 0 1 8 4zM3.732 5.732a.5.5 0 0 1 .707 0l.915.914a.5.5 0 1 1-.708.708l-.914-.915a.5.5 0 0 1 0-.707zM2 10a.5.5 0 0 1 .5-.5h1.586a.5.5 0 0 1 0 1H2.5A.5.5 0 0 1 2 10zm9.5 0a.5.5 0 0 1 .5-.5h1.5a.5.5 0 0 1 0 1H12a.5.5 0 0 1-.5-.5zm.754-4.246a.389.389 0 0 0-.527-.02L7.547 9.31a.91.91 0 1 0 1.302 1.258l3.434-4.297a.389.389 0 0 0-.029-.518z"/>
|
|
||||||
<path fill-rule="evenodd" d="M0 10a8 8 0 1 1 15.547 2.661c-.442 1.253-1.845 1.602-2.932 1.25C11.309 13.488 9.475 13 8 13c-1.474 0-3.31.488-4.615.911-1.087.352-2.49.003-2.932-1.25A7.988 7.988 0 0 1 0 10zm8-7a7 7 0 0 0-6.603 9.329c.203.575.923.876 1.68.63C4.397 12.533 6.358 12 8 12s3.604.532 4.923.96c.757.245 1.477-.056 1.68-.631A7 7 0 0 0 8 3z"/>
|
|
||||||
</svg>
|
|
||||||
<span class="sidebar-link-text">Metrics</span>
|
|
||||||
</a>
|
|
||||||
<a href="{{ url_for(endpoint="ui.sites_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.sites_dashboard" %}active{% endif %}" data-tooltip="Sites">
|
<a href="{{ url_for(endpoint="ui.sites_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.sites_dashboard" %}active{% endif %}" data-tooltip="Sites">
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||||
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472zM3.82 11a13.652 13.652 0 0 1-.312-2.5h-2.49c.062.89.291 1.733.656 2.5H3.82zm6.853 3.472A7.024 7.024 0 0 0 13.745 12H11.91a9.27 9.27 0 0 1-.64 1.539 6.688 6.688 0 0 1-.597.933zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855.173-.324.33-.682.468-1.068H8.5zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.65 13.65 0 0 1-.312 2.5zm2.802-3.5a6.959 6.959 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5h2.49zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7.024 7.024 0 0 0-3.072-2.472c.218.284.418.598.597.933zM10.855 4a7.966 7.966 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4h2.355z"/>
|
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472zM3.82 11a13.652 13.652 0 0 1-.312-2.5h-2.49c.062.89.291 1.733.656 2.5H3.82zm6.853 3.472A7.024 7.024 0 0 0 13.745 12H11.91a9.27 9.27 0 0 1-.64 1.539 6.688 6.688 0 0 1-.597.933zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855.173-.324.33-.682.468-1.068H8.5zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.65 13.65 0 0 1-.312 2.5zm2.802-3.5a6.959 6.959 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5h2.49zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7.024 7.024 0 0 0-3.072-2.472c.218.284.418.598.597.933zM10.855 4a7.966 7.966 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4h2.355z"/>
|
||||||
</svg>
|
</svg>
|
||||||
<span class="sidebar-link-text">Sites</span>
|
<span class="sidebar-link-text">Sites</span>
|
||||||
</a>
|
</a>
|
||||||
|
<a href="{{ url_for(endpoint="ui.cluster_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.cluster_dashboard" %}active{% endif %}" data-tooltip="Cluster">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||||
|
<path d="M7.752.066a.5.5 0 0 1 .496 0l3.75 2.143a.5.5 0 0 1 .252.434v3.995l3.498 2A.5.5 0 0 1 16 9.07v4.286a.5.5 0 0 1-.252.434l-3.75 2.143a.5.5 0 0 1-.496 0l-3.502-2-3.502 2.001a.5.5 0 0 1-.496 0l-3.75-2.143A.5.5 0 0 1 0 13.357V9.071a.5.5 0 0 1 .252-.434L3.75 6.638V2.643a.5.5 0 0 1 .252-.434L7.752.066ZM4.25 7.504 1.508 9.071l2.742 1.567 2.742-1.567L4.25 7.504ZM7.5 9.933l-2.75 1.571v3.134l2.75-1.571V9.933Zm1 3.134 2.75 1.571v-3.134L8.5 9.933v3.134Zm.508-3.996 2.742 1.567 2.742-1.567-2.742-1.567-2.742 1.567Zm2.242-2.433V3.504L8.5 5.076V8.21l2.75-1.572ZM7.5 8.21V5.076L4.75 3.504v3.134L7.5 8.21ZM5.258 2.643 8 4.21l2.742-1.567L8 1.076 5.258 2.643ZM15 9.933l-2.75 1.571v3.134L15 13.067V9.933ZM3.75 14.638v-3.134L1 9.933v3.134l2.75 1.571Z"/>
|
||||||
|
</svg>
|
||||||
|
<span class="sidebar-link-text">Cluster</span>
|
||||||
|
</a>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% if website_hosting_nav %}
|
{% if website_hosting_nav %}
|
||||||
<a href="{{ url_for(endpoint="ui.website_domains_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.website_domains_dashboard" %}active{% endif %}" data-tooltip="Domains">
|
<a href="{{ url_for(endpoint="ui.website_domains_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.website_domains_dashboard" %}active{% endif %}" data-tooltip="Domains">
|
||||||
@@ -219,6 +224,13 @@
|
|||||||
</a>
|
</a>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% if can_manage_iam %}
|
{% if can_manage_iam %}
|
||||||
|
<a href="{{ url_for(endpoint="ui.metrics_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.metrics_dashboard" %}active{% endif %}" data-tooltip="Metrics">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||||
|
<path d="M8 4a.5.5 0 0 1 .5.5V6a.5.5 0 0 1-1 0V4.5A.5.5 0 0 1 8 4zM3.732 5.732a.5.5 0 0 1 .707 0l.915.914a.5.5 0 1 1-.708.708l-.914-.915a.5.5 0 0 1 0-.707zM2 10a.5.5 0 0 1 .5-.5h1.586a.5.5 0 0 1 0 1H2.5A.5.5 0 0 1 2 10zm9.5 0a.5.5 0 0 1 .5-.5h1.5a.5.5 0 0 1 0 1H12a.5.5 0 0 1-.5-.5zm.754-4.246a.389.389 0 0 0-.527-.02L7.547 9.31a.91.91 0 1 0 1.302 1.258l3.434-4.297a.389.389 0 0 0-.029-.518z"/>
|
||||||
|
<path fill-rule="evenodd" d="M0 10a8 8 0 1 1 15.547 2.661c-.442 1.253-1.845 1.602-2.932 1.25C11.309 13.488 9.475 13 8 13c-1.474 0-3.31.488-4.615.911-1.087.352-2.49.003-2.932-1.25A7.988 7.988 0 0 1 0 10zm8-7a7 7 0 0 0-6.603 9.329c.203.575.923.876 1.68.63C4.397 12.533 6.358 12 8 12s3.604.532 4.923.96c.757.245 1.477-.056 1.68-.631A7 7 0 0 0 8 3z"/>
|
||||||
|
</svg>
|
||||||
|
<span class="sidebar-link-text">Metrics</span>
|
||||||
|
</a>
|
||||||
<a href="{{ url_for(endpoint="ui.system_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.system_dashboard" %}active{% endif %}" data-tooltip="System">
|
<a href="{{ url_for(endpoint="ui.system_dashboard") }}" class="sidebar-link {% if current_endpoint == "ui.system_dashboard" %}active{% endif %}" data-tooltip="System">
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||||
<path d="M9.405 1.05c-.413-1.4-2.397-1.4-2.81 0l-.1.34a1.464 1.464 0 0 1-2.105.872l-.31-.17c-1.283-.698-2.686.705-1.987 1.987l.169.311c.446.82.023 1.841-.872 2.105l-.34.1c-1.4.413-1.4 2.397 0 2.81l.34.1a1.464 1.464 0 0 1 .872 2.105l-.17.31c-.698 1.283.705 2.686 1.987 1.987l.311-.169a1.464 1.464 0 0 1 2.105.872l.1.34c.413 1.4 2.397 1.4 2.81 0l.1-.34a1.464 1.464 0 0 1 2.105-.872l.31.17c1.283.698 2.686-.705 1.987-1.987l-.169-.311a1.464 1.464 0 0 1 .872-2.105l.34-.1c1.4-.413 1.4-2.397 0-2.81l-.34-.1a1.464 1.464 0 0 1-.872-2.105l.17-.31c.698-1.283-.705-2.686-1.987-1.987l-.311.169a1.464 1.464 0 0 1-2.105-.872l-.1-.34zM8 10.93a2.929 2.929 0 1 1 0-5.86 2.929 2.929 0 0 1 0 5.858z"/>
|
<path d="M9.405 1.05c-.413-1.4-2.397-1.4-2.81 0l-.1.34a1.464 1.464 0 0 1-2.105.872l-.31-.17c-1.283-.698-2.686.705-1.987 1.987l.169.311c.446.82.023 1.841-.872 2.105l-.34.1c-1.4.413-1.4 2.397 0 2.81l.34.1a1.464 1.464 0 0 1 .872 2.105l-.17.31c-.698 1.283.705 2.686 1.987 1.987l.311-.169a1.464 1.464 0 0 1 2.105.872l.1.34c.413 1.4 2.397 1.4 2.81 0l.1-.34a1.464 1.464 0 0 1 2.105-.872l.31.17c1.283.698 2.686-.705 1.987-1.987l-.169-.311a1.464 1.464 0 0 1 .872-2.105l.34-.1c1.4-.413 1.4-2.397 0-2.81l-.34-.1a1.464 1.464 0 0 1-.872-2.105l.17-.31c.698-1.283-.705-2.686-1.987-1.987l-.311.169a1.464 1.464 0 0 1-2.105-.872l-.1-.34zM8 10.93a2.929 2.929 0 1 1 0-5.86 2.929 2.929 0 0 1 0 5.858z"/>
|
||||||
|
|||||||
@@ -19,11 +19,11 @@
|
|||||||
<div>
|
<div>
|
||||||
<h1 class="h3 fw-bold mb-1">{{ bucket_name }}</h1>
|
<h1 class="h3 fw-bold mb-1">{{ bucket_name }}</h1>
|
||||||
<div class="d-flex align-items-center gap-2">
|
<div class="d-flex align-items-center gap-2">
|
||||||
<span class="badge {% if versioning_enabled %}text-bg-success{% else %}text-bg-secondary{% endif %} rounded-pill">
|
<span class="badge {% if versioning_enabled %}text-bg-success{% elif versioning_suspended %}text-bg-warning{% else %}text-bg-secondary{% endif %} rounded-pill">
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||||
<path d="M8 16a2 2 0 0 0 2-2H6a2 2 0 0 0 2 2zm.995-14.901a1 1 0 1 0-1.99 0A5.002 5.002 0 0 0 3 6c0 1.098-.5 6-2 7h14c-1.5-1-2-5.902-2-7 0-2.42-1.72-4.44-4.005-4.901z"/>
|
<path d="M8 16a2 2 0 0 0 2-2H6a2 2 0 0 0 2 2zm.995-14.901a1 1 0 1 0-1.99 0A5.002 5.002 0 0 0 3 6c0 1.098-.5 6-2 7h14c-1.5-1-2-5.902-2-7 0-2.42-1.72-4.44-4.005-4.901z"/>
|
||||||
</svg>
|
</svg>
|
||||||
{% if versioning_enabled %}Versioning On{% else %}Versioning Off{% endif %}
|
{% if versioning_enabled %}Versioning On{% elif versioning_suspended %}Versioning Suspended{% else %}Versioning Off{% endif %}
|
||||||
</span>
|
</span>
|
||||||
<span class="text-muted small" id="object-count-badge">
|
<span class="text-muted small" id="object-count-badge">
|
||||||
<span class="spinner-border spinner-border-sm" role="status" style="width: 0.75rem; height: 0.75rem;"></span>
|
<span class="spinner-border spinner-border-sm" role="status" style="width: 0.75rem; height: 0.75rem;"></span>
|
||||||
@@ -626,6 +626,16 @@
|
|||||||
<p class="mb-0 small">All previous versions of objects are preserved. You can roll back accidental changes or deletions at any time.</p>
|
<p class="mb-0 small">All previous versions of objects are preserved. You can roll back accidental changes or deletions at any time.</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
{% elif versioning_suspended %}
|
||||||
|
<div class="alert alert-warning d-flex align-items-start mb-4" role="alert">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="me-2 flex-shrink-0" viewBox="0 0 16 16">
|
||||||
|
<path d="M5.5 3.5A1.5 1.5 0 0 1 7 5v6a1.5 1.5 0 0 1-3 0V5a1.5 1.5 0 0 1 1.5-1.5zm5 0A1.5 1.5 0 0 1 12 5v6a1.5 1.5 0 0 1-3 0V5a1.5 1.5 0 0 1 1.5-1.5z"/>
|
||||||
|
</svg>
|
||||||
|
<div>
|
||||||
|
<strong>Versioning is suspended</strong>
|
||||||
|
<p class="mb-0 small">New uploads overwrite existing objects, but previously archived versions are still retained. Re-enable versioning to start preserving new versions again.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
{% else %}
|
{% else %}
|
||||||
<div class="alert alert-secondary d-flex align-items-start mb-4" role="alert">
|
<div class="alert alert-secondary d-flex align-items-start mb-4" role="alert">
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="me-2 flex-shrink-0" viewBox="0 0 16 16">
|
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="me-2 flex-shrink-0" viewBox="0 0 16 16">
|
||||||
@@ -633,8 +643,8 @@
|
|||||||
<path d="M4.646 4.646a.5.5 0 0 1 .708 0L8 7.293l2.646-2.647a.5.5 0 0 1 .708.708L8.707 8l2.647 2.646a.5.5 0 0 1-.708.708L8 8.707l-2.646 2.647a.5.5 0 0 1-.708-.708L7.293 8 4.646 5.354a.5.5 0 0 1 0-.708z"/>
|
<path d="M4.646 4.646a.5.5 0 0 1 .708 0L8 7.293l2.646-2.647a.5.5 0 0 1 .708.708L8.707 8l2.647 2.646a.5.5 0 0 1-.708.708L8 8.707l-2.646 2.647a.5.5 0 0 1-.708-.708L7.293 8 4.646 5.354a.5.5 0 0 1 0-.708z"/>
|
||||||
</svg>
|
</svg>
|
||||||
<div>
|
<div>
|
||||||
<strong>Versioning is suspended</strong>
|
<strong>Versioning is disabled</strong>
|
||||||
<p class="mb-0 small">New object uploads overwrite existing objects. Enable versioning to preserve previous versions.</p>
|
<p class="mb-0 small">This bucket has never had versioning enabled. Enable it to preserve previous versions of every object.</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
@@ -869,9 +879,10 @@
|
|||||||
<h6 class="small fw-semibold mb-3">Current Usage</h6>
|
<h6 class="small fw-semibold mb-3">Current Usage</h6>
|
||||||
<div class="row g-3">
|
<div class="row g-3">
|
||||||
<div class="col-6">
|
<div class="col-6">
|
||||||
<div class="border rounded p-3 text-center">
|
<div class="border rounded p-3 text-center" data-usage-objects data-total-objects="{{ total_objects }}" data-max-objects="{% if has_max_objects %}{{ max_objects }}{% endif %}">
|
||||||
<div class="fs-4 fw-bold text-primary">{{ total_objects }}</div>
|
<div class="fs-4 fw-bold text-primary" data-usage-objects-value>{{ total_objects }}</div>
|
||||||
<div class="small text-muted">Total Objects</div>
|
<div class="small text-muted">Total Objects</div>
|
||||||
|
<div data-usage-objects-limit>
|
||||||
{% if has_max_objects %}
|
{% if has_max_objects %}
|
||||||
<div class="progress mt-2" style="height: 4px;">
|
<div class="progress mt-2" style="height: 4px;">
|
||||||
{% if max_objects > 0 %}{% set obj_pct = total_objects / max_objects * 100 | int %}{% else %}{% set obj_pct = 0 %}{% endif %}
|
{% if max_objects > 0 %}{% set obj_pct = total_objects / max_objects * 100 | int %}{% else %}{% set obj_pct = 0 %}{% endif %}
|
||||||
@@ -881,6 +892,7 @@
|
|||||||
{% else %}
|
{% else %}
|
||||||
<div class="small text-muted mt-2">No limit</div>
|
<div class="small text-muted mt-2">No limit</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
</div>
|
||||||
{% if version_count > 0 %}
|
{% if version_count > 0 %}
|
||||||
<div class="small text-muted mt-1">
|
<div class="small text-muted mt-1">
|
||||||
<span class="text-body-secondary">({{ current_objects }} current + {{ version_count }} versions)</span>
|
<span class="text-body-secondary">({{ current_objects }} current + {{ version_count }} versions)</span>
|
||||||
@@ -889,9 +901,10 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="col-6">
|
<div class="col-6">
|
||||||
<div class="border rounded p-3 text-center">
|
<div class="border rounded p-3 text-center" data-usage-bytes data-total-bytes="{{ total_bytes }}" data-max-bytes="{% if has_max_bytes %}{{ max_bytes }}{% endif %}">
|
||||||
<div class="fs-4 fw-bold text-primary">{{ total_bytes | filesizeformat }}</div>
|
<div class="fs-4 fw-bold text-primary" data-usage-bytes-value>{{ total_bytes | filesizeformat }}</div>
|
||||||
<div class="small text-muted">Total Storage</div>
|
<div class="small text-muted">Total Storage</div>
|
||||||
|
<div data-usage-bytes-limit>
|
||||||
{% if has_max_bytes %}
|
{% if has_max_bytes %}
|
||||||
<div class="progress mt-2" style="height: 4px;">
|
<div class="progress mt-2" style="height: 4px;">
|
||||||
{% if max_bytes > 0 %}{% set bytes_pct = total_bytes / max_bytes * 100 | int %}{% else %}{% set bytes_pct = 0 %}{% endif %}
|
{% if max_bytes > 0 %}{% set bytes_pct = total_bytes / max_bytes * 100 | int %}{% else %}{% set bytes_pct = 0 %}{% endif %}
|
||||||
@@ -901,6 +914,7 @@
|
|||||||
{% else %}
|
{% else %}
|
||||||
<div class="small text-muted mt-2">No limit</div>
|
<div class="small text-muted mt-2">No limit</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
</div>
|
||||||
{% if version_bytes > 0 %}
|
{% if version_bytes > 0 %}
|
||||||
<div class="small text-muted mt-1">
|
<div class="small text-muted mt-1">
|
||||||
<span class="text-body-secondary">({{ current_bytes | filesizeformat }} current + {{ version_bytes | filesizeformat }} versions)</span>
|
<span class="text-body-secondary">({{ current_bytes | filesizeformat }} current + {{ version_bytes | filesizeformat }} versions)</span>
|
||||||
|
|||||||
461
crates/myfsio-server/templates/cluster.html
Normal file
461
crates/myfsio-server/templates/cluster.html
Normal file
@@ -0,0 +1,461 @@
|
|||||||
|
{% extends "base.html" %}
|
||||||
|
|
||||||
|
{% block title %}Cluster - S3 Compatible Storage{% endblock %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<div class="page-header d-flex justify-content-between align-items-center mb-4">
|
||||||
|
<div>
|
||||||
|
<p class="text-uppercase text-muted small mb-1">Cluster Overview</p>
|
||||||
|
<h1 class="h3 mb-1 d-flex align-items-center gap-2">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="28" height="28" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||||
|
<path d="M7.752.066a.5.5 0 0 1 .496 0l3.75 2.143a.5.5 0 0 1 .252.434v3.995l3.498 2A.5.5 0 0 1 16 9.07v4.286a.5.5 0 0 1-.252.434l-3.75 2.143a.5.5 0 0 1-.496 0l-3.502-2-3.502 2.001a.5.5 0 0 1-.496 0l-3.75-2.143A.5.5 0 0 1 0 13.357V9.071a.5.5 0 0 1 .252-.434L3.75 6.638V2.643a.5.5 0 0 1 .252-.434L7.752.066ZM4.25 7.504 1.508 9.071l2.742 1.567 2.742-1.567L4.25 7.504ZM7.5 9.933l-2.75 1.571v3.134l2.75-1.571V9.933Zm1 3.134 2.75 1.571v-3.134L8.5 9.933v3.134Zm.508-3.996 2.742 1.567 2.742-1.567-2.742-1.567-2.742 1.567Zm2.242-2.433V3.504L8.5 5.076V8.21l2.75-1.572ZM7.5 8.21V5.076L4.75 3.504v3.134L7.5 8.21ZM5.258 2.643 8 4.21l2.742-1.567L8 1.076 5.258 2.643ZM15 9.933l-2.75 1.571v3.134L15 13.067V9.933ZM3.75 14.638v-3.134L1 9.933v3.134l2.75 1.571Z"/>
|
||||||
|
</svg>
|
||||||
|
Cluster
|
||||||
|
</h1>
|
||||||
|
<p class="text-muted mb-0 mt-1">Live view across this site and every registered peer.</p>
|
||||||
|
</div>
|
||||||
|
<div class="d-flex align-items-center gap-2">
|
||||||
|
<span class="badge bg-success bg-opacity-10 text-success fs-6 px-3 py-2" id="cluster-online-badge">
|
||||||
|
{{ cluster_online_count }} / {{ cluster_total_count }} online
|
||||||
|
</span>
|
||||||
|
<span class="text-muted small d-none d-md-inline" id="cluster-updated-at" title="Last refresh">just now</span>
|
||||||
|
<button type="button" class="btn btn-outline-secondary btn-sm d-flex align-items-center gap-1" id="cluster-refresh-btn" title="Refresh now (bypass 10s cache)">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16" id="cluster-refresh-icon">
|
||||||
|
<path fill-rule="evenodd" d="M8 3a5 5 0 1 0 4.546 2.914.5.5 0 0 1 .908-.417A6 6 0 1 1 8 2v1z"/>
|
||||||
|
<path d="M8 4.466V.534a.25.25 0 0 1 .41-.192l2.36 1.966c.12.1.12.284 0 .384L8.41 4.658A.25.25 0 0 1 8 4.466z"/>
|
||||||
|
</svg>
|
||||||
|
<span>Refresh</span>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="row g-3 mb-4">
|
||||||
|
<div class="col-md-3 col-sm-6">
|
||||||
|
<div class="card shadow-sm border-0 h-100" style="border-radius: 1rem;">
|
||||||
|
<div class="card-body d-flex align-items-center gap-3">
|
||||||
|
<div class="d-flex align-items-center justify-content-center rounded-3 bg-primary bg-opacity-10 text-primary" style="width:44px;height:44px;flex-shrink:0;">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="22" height="22" fill="currentColor" viewBox="0 0 16 16">
|
||||||
|
<path d="M7.752.066a.5.5 0 0 1 .496 0l3.75 2.143a.5.5 0 0 1 .252.434v3.995l3.498 2A.5.5 0 0 1 16 9.07v4.286a.5.5 0 0 1-.252.434l-3.75 2.143a.5.5 0 0 1-.496 0l-3.502-2-3.502 2.001a.5.5 0 0 1-.496 0l-3.75-2.143A.5.5 0 0 1 0 13.357V9.071a.5.5 0 0 1 .252-.434L3.75 6.638V2.643a.5.5 0 0 1 .252-.434L7.752.066Z"/>
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
|
<div class="flex-grow-1">
|
||||||
|
<div class="text-uppercase text-muted small">Sites</div>
|
||||||
|
<div class="h3 mb-0" id="cluster-total-sites">{{ cluster_total_count }}</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-3 col-sm-6">
|
||||||
|
<div class="card shadow-sm border-0 h-100" style="border-radius: 1rem;">
|
||||||
|
<div class="card-body d-flex align-items-center gap-3">
|
||||||
|
<div class="d-flex align-items-center justify-content-center rounded-3 bg-info bg-opacity-10 text-info" style="width:44px;height:44px;flex-shrink:0;">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="22" height="22" fill="currentColor" viewBox="0 0 16 16">
|
||||||
|
<path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/>
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
|
<div class="flex-grow-1">
|
||||||
|
<div class="text-uppercase text-muted small">Buckets</div>
|
||||||
|
<div class="h3 mb-0" id="cluster-total-buckets">{{ cluster_total_buckets }}</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-3 col-sm-6">
|
||||||
|
<div class="card shadow-sm border-0 h-100" style="border-radius: 1rem;">
|
||||||
|
<div class="card-body d-flex align-items-center gap-3">
|
||||||
|
<div class="d-flex align-items-center justify-content-center rounded-3 bg-warning bg-opacity-10 text-warning" style="width:44px;height:44px;flex-shrink:0;">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="22" height="22" fill="currentColor" viewBox="0 0 16 16">
|
||||||
|
<path d="M14 14V4.5L9.5 0H4a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h8a2 2 0 0 0 2-2zM9.5 3A1.5 1.5 0 0 0 11 4.5h2V14a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1h5.5z"/>
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
|
<div class="flex-grow-1">
|
||||||
|
<div class="text-uppercase text-muted small">Objects</div>
|
||||||
|
<div class="h3 mb-0" id="cluster-total-objects">{{ cluster_total_objects }}</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-3 col-sm-6">
|
||||||
|
<div class="card shadow-sm border-0 h-100" style="border-radius: 1rem;">
|
||||||
|
<div class="card-body d-flex align-items-center gap-3">
|
||||||
|
<div class="d-flex align-items-center justify-content-center rounded-3 bg-success bg-opacity-10 text-success" style="width:44px;height:44px;flex-shrink:0;">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="22" height="22" fill="currentColor" viewBox="0 0 16 16">
|
||||||
|
<path d="M0 10a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8H0v2zm1.5 1a.5.5 0 1 1 0-1 .5.5 0 0 1 0 1zm2 0a.5.5 0 1 1 0-1 .5.5 0 0 1 0 1zM0 4a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v3H0V4z"/>
|
||||||
|
<path d="M1.5 6a.5.5 0 1 0 0-1 .5.5 0 0 0 0 1zm2 0a.5.5 0 1 0 0-1 .5.5 0 0 0 0 1z"/>
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
|
<div class="flex-grow-1">
|
||||||
|
<div class="text-uppercase text-muted small">Size</div>
|
||||||
|
<div class="h3 mb-0" id="cluster-total-size" data-bytes="{{ cluster_total_size_bytes }}">{{ cluster_total_size_bytes }}</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="row g-4" id="cluster-sites-row">
|
||||||
|
{% for site in cluster_sites %}
|
||||||
|
<div class="col-xl-6">
|
||||||
|
<div class="card shadow-sm border-0 h-100 site-card" data-site-id="{{ site.site_id }}" style="border-radius: 1rem;">
|
||||||
|
<div class="card-body p-4">
|
||||||
|
<div class="d-flex align-items-start justify-content-between mb-3">
|
||||||
|
<div class="flex-grow-1 min-w-0">
|
||||||
|
<div class="d-flex align-items-center gap-2 mb-1 flex-wrap">
|
||||||
|
<span class="badge bg-success bg-opacity-10 text-success site-status-online {% if not site.online %}d-none{% endif %}">
|
||||||
|
<span class="d-inline-block rounded-circle bg-success me-1" style="width:6px;height:6px;"></span>online
|
||||||
|
</span>
|
||||||
|
<span class="badge bg-danger bg-opacity-10 text-danger site-status-offline {% if site.online %}d-none{% endif %}">
|
||||||
|
<span class="d-inline-block rounded-circle bg-danger me-1" style="width:6px;height:6px;"></span>offline
|
||||||
|
</span>
|
||||||
|
<span class="badge bg-warning bg-opacity-10 text-warning site-status-stale {% if not site.stale %}d-none{% endif %}" title="Could not reach peer">stale</span>
|
||||||
|
{% if site.is_local %}
|
||||||
|
<span class="badge bg-primary bg-opacity-10 text-primary">this site</span>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
<h5 class="fw-semibold mb-0 text-truncate">
|
||||||
|
{% if site.display_name and site.display_name != "" %}{{ site.display_name }}{% else %}{{ site.site_id }}{% endif %}
|
||||||
|
</h5>
|
||||||
|
<div class="text-muted small">
|
||||||
|
<span class="font-monospace">{{ site.site_id }}</span>
|
||||||
|
{% if site.region %} · {{ site.region }}{% elif site.registered_region %} · {{ site.registered_region }}{% endif %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% if site.endpoint %}
|
||||||
|
<code class="small text-muted text-end ms-2" style="word-break:break-all;">{{ site.endpoint }}</code>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="site-online-content {% if not site.online %}d-none{% endif %}">
|
||||||
|
<div class="row g-3 mb-3">
|
||||||
|
<div class="col-4">
|
||||||
|
<div class="text-uppercase text-muted small mb-1 d-flex align-items-center gap-1">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="11" height="11" fill="currentColor" viewBox="0 0 16 16"><path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/></svg>
|
||||||
|
Buckets
|
||||||
|
</div>
|
||||||
|
<div class="h4 mb-0 site-buckets">{{ site.buckets | default(value=0) }}</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-4">
|
||||||
|
<div class="text-uppercase text-muted small mb-1 d-flex align-items-center gap-1">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="11" height="11" fill="currentColor" viewBox="0 0 16 16"><path d="M14 14V4.5L9.5 0H4a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h8a2 2 0 0 0 2-2z"/></svg>
|
||||||
|
Objects
|
||||||
|
</div>
|
||||||
|
<div class="h4 mb-0 site-objects">{{ site.objects | default(value=0) }}</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-4">
|
||||||
|
<div class="text-uppercase text-muted small mb-1 d-flex align-items-center gap-1">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="11" height="11" fill="currentColor" viewBox="0 0 16 16"><path d="M0 10a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8H0v2zM0 4a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v3H0V4z"/></svg>
|
||||||
|
Size
|
||||||
|
</div>
|
||||||
|
<div class="h4 mb-0 site-size" data-bytes="{{ site.size_bytes | default(value=0) }}">{{ site.size_bytes | default(value=0) }}</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{% if site.capacity %}
|
||||||
|
<div class="mb-3">
|
||||||
|
<div class="d-flex justify-content-between align-items-center mb-1">
|
||||||
|
<span class="text-uppercase text-muted small">Disk Capacity</span>
|
||||||
|
<span class="small text-muted">
|
||||||
|
<span class="site-disk-used" data-bytes="0">0</span> / <span class="site-disk-total" data-bytes="{{ site.capacity.total_bytes | default(value=0) }}">{{ site.capacity.total_bytes | default(value=0) }}</span>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div class="progress" style="height:6px;border-radius:3px;">
|
||||||
|
<div class="progress-bar bg-primary site-disk-bar" role="progressbar" style="width:0%;" data-total="{{ site.capacity.total_bytes | default(value=0) }}" data-available="{{ site.capacity.available_bytes | default(value=0) }}"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% if site.system and site.system.cpu_percent is defined %}
|
||||||
|
<div class="mb-3">
|
||||||
|
<div class="text-uppercase text-muted small mb-2">System</div>
|
||||||
|
<div class="d-flex flex-column gap-2">
|
||||||
|
<div>
|
||||||
|
<div class="d-flex justify-content-between small mb-1">
|
||||||
|
<span class="text-muted">CPU</span>
|
||||||
|
<span class="site-cpu-label">{{ site.system.cpu_percent | default(value=0) }}%</span>
|
||||||
|
</div>
|
||||||
|
<div class="progress" style="height:4px;border-radius:2px;">
|
||||||
|
<div class="progress-bar site-cpu-bar" role="progressbar" style="width:{{ site.system.cpu_percent | default(value=0) }}%;"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<div class="d-flex justify-content-between small mb-1">
|
||||||
|
<span class="text-muted">Memory</span>
|
||||||
|
<span class="site-mem-label">{{ site.system.memory_percent | default(value=0) }}%</span>
|
||||||
|
</div>
|
||||||
|
<div class="progress" style="height:4px;border-radius:2px;">
|
||||||
|
<div class="progress-bar site-mem-bar" role="progressbar" style="width:{{ site.system.memory_percent | default(value=0) }}%;"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<div class="d-flex justify-content-between small mb-1">
|
||||||
|
<span class="text-muted">Disk</span>
|
||||||
|
<span class="site-diskpct-label">{{ site.system.disk_percent | default(value=0) }}%</span>
|
||||||
|
</div>
|
||||||
|
<div class="progress" style="height:4px;border-radius:2px;">
|
||||||
|
<div class="progress-bar site-diskpct-bar" role="progressbar" style="width:{{ site.system.disk_percent | default(value=0) }}%;"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% if site.sync %}
|
||||||
|
<div class="d-flex align-items-center justify-content-between border-top pt-3">
|
||||||
|
<div class="d-flex align-items-center gap-2 small">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||||
|
<path fill-rule="evenodd" d="M8 3a5 5 0 1 0 4.546 2.914.5.5 0 0 1 .908-.417A6 6 0 1 1 8 2v1z"/>
|
||||||
|
<path d="M8 4.466V.534a.25.25 0 0 1 .41-.192l2.36 1.966c.12.1.12.284 0 .384L8.41 4.658A.25.25 0 0 1 8 4.466z"/>
|
||||||
|
</svg>
|
||||||
|
<span class="text-muted">Sync</span>
|
||||||
|
<span class="site-sync-label">
|
||||||
|
{% if site.sync.last_sync_at %}
|
||||||
|
<span data-last-sync-at="{{ site.sync.last_sync_at }}">last sync <span class="last-sync-rel">just now</span></span>
|
||||||
|
{% else %}
|
||||||
|
<span class="text-muted">no sync yet</span>
|
||||||
|
{% endif %}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<span class="badge bg-danger bg-opacity-10 text-danger site-sync-errors {% if not site.sync.errors or site.sync.errors == 0 %}d-none{% endif %}">
|
||||||
|
<span class="site-sync-errors-count">{{ site.sync.errors | default(value=0) }}</span> err
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="site-offline-content {% if site.online %}d-none{% endif %}">
|
||||||
|
<div class="alert alert-light border-0 mb-0 py-2 px-3 small">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1 text-warning" viewBox="0 0 16 16">
|
||||||
|
<path d="M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z"/>
|
||||||
|
</svg>
|
||||||
|
<span class="site-offline-message">{% if site.error %}{{ site.error }}{% else %}Peer unreachable.{% endif %}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
{% if cluster_total_count <= 1 %}
|
||||||
|
<div class="col-xl-6">
|
||||||
|
<a href="{{ url_for(endpoint='ui.sites_dashboard') }}" class="card shadow-sm border-0 h-100 text-decoration-none text-reset" style="border-radius: 1rem; border: 2px dashed var(--bs-border-color) !important;">
|
||||||
|
<div class="card-body d-flex flex-column align-items-center justify-content-center text-center p-5">
|
||||||
|
<div class="d-flex align-items-center justify-content-center rounded-circle bg-primary bg-opacity-10 text-primary mb-3" style="width:64px;height:64px;">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" fill="currentColor" viewBox="0 0 16 16">
|
||||||
|
<path d="M8 4a.5.5 0 0 1 .5.5v3h3a.5.5 0 0 1 0 1h-3v3a.5.5 0 0 1-1 0v-3h-3a.5.5 0 0 1 0-1h3v-3A.5.5 0 0 1 8 4z"/>
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
|
<h5 class="fw-semibold mb-1">Add a peer site</h5>
|
||||||
|
<p class="text-muted small mb-0">Register another MyFSIO instance to see it appear here side-by-side.</p>
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
(function () {
|
||||||
|
function fmtBytes(n) {
|
||||||
|
if (!n || n < 0) return "0 B";
|
||||||
|
var u = ["B", "KB", "MB", "GB", "TB", "PB"];
|
||||||
|
var i = 0;
|
||||||
|
var v = n;
|
||||||
|
while (v >= 1024 && i < u.length - 1) { v /= 1024; i++; }
|
||||||
|
return v.toFixed(i === 0 ? 0 : (v >= 100 ? 0 : 1)) + " " + u[i];
|
||||||
|
}
|
||||||
|
function fmtRel(ts) {
|
||||||
|
var diff = Math.max(0, Math.floor(Date.now() / 1000 - ts));
|
||||||
|
if (diff < 60) return diff + "s ago";
|
||||||
|
if (diff < 3600) return Math.floor(diff / 60) + "m ago";
|
||||||
|
if (diff < 86400) return Math.floor(diff / 3600) + "h ago";
|
||||||
|
return Math.floor(diff / 86400) + "d ago";
|
||||||
|
}
|
||||||
|
function pctColor(p) {
|
||||||
|
if (p >= 80) return "bg-danger";
|
||||||
|
if (p >= 60) return "bg-warning";
|
||||||
|
return "bg-success";
|
||||||
|
}
|
||||||
|
function applyBytesFormat(root) {
|
||||||
|
(root || document).querySelectorAll("[data-bytes]").forEach(function (el) {
|
||||||
|
var n = parseInt(el.getAttribute("data-bytes"), 10);
|
||||||
|
if (!isNaN(n)) el.textContent = fmtBytes(n);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
function applyDiskBars(root) {
|
||||||
|
(root || document).querySelectorAll(".site-disk-bar").forEach(function (bar) {
|
||||||
|
var total = parseFloat(bar.getAttribute("data-total")) || 0;
|
||||||
|
var avail = parseFloat(bar.getAttribute("data-available")) || 0;
|
||||||
|
var used = Math.max(0, total - avail);
|
||||||
|
var pct = total > 0 ? (used / total) * 100 : 0;
|
||||||
|
bar.style.width = pct.toFixed(1) + "%";
|
||||||
|
bar.classList.remove("bg-success", "bg-warning", "bg-danger", "bg-primary");
|
||||||
|
bar.classList.add(pctColor(pct));
|
||||||
|
var card = bar.closest(".site-card");
|
||||||
|
if (card) {
|
||||||
|
var usedEl = card.querySelector(".site-disk-used");
|
||||||
|
if (usedEl) {
|
||||||
|
usedEl.setAttribute("data-bytes", String(Math.floor(used)));
|
||||||
|
usedEl.textContent = fmtBytes(used);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
function applyPctBars(root) {
|
||||||
|
var pairs = [
|
||||||
|
[".site-cpu-bar", ".site-cpu-label"],
|
||||||
|
[".site-mem-bar", ".site-mem-label"],
|
||||||
|
[".site-diskpct-bar", ".site-diskpct-label"],
|
||||||
|
];
|
||||||
|
pairs.forEach(function (sel) {
|
||||||
|
(root || document).querySelectorAll(sel[0]).forEach(function (bar) {
|
||||||
|
var label = bar.closest(".site-card").querySelector(sel[1]);
|
||||||
|
var pct = parseFloat(label ? label.textContent : "0") || 0;
|
||||||
|
bar.classList.remove("bg-success", "bg-warning", "bg-danger");
|
||||||
|
bar.classList.add(pctColor(pct));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
function refreshRel() {
|
||||||
|
document.querySelectorAll("[data-last-sync-at]").forEach(function (el) {
|
||||||
|
var ts = parseFloat(el.getAttribute("data-last-sync-at"));
|
||||||
|
var span = el.querySelector(".last-sync-rel");
|
||||||
|
if (span && !isNaN(ts)) span.textContent = fmtRel(ts);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateCard(card, site) {
|
||||||
|
if (!card) return;
|
||||||
|
var online = !!site.online;
|
||||||
|
var stale = !!site.stale;
|
||||||
|
|
||||||
|
function toggle(sel, show) {
|
||||||
|
var el = card.querySelector(sel);
|
||||||
|
if (el) el.classList.toggle("d-none", !show);
|
||||||
|
}
|
||||||
|
toggle(".site-status-online", online);
|
||||||
|
toggle(".site-status-offline", !online);
|
||||||
|
toggle(".site-status-stale", stale);
|
||||||
|
toggle(".site-online-content", online);
|
||||||
|
toggle(".site-offline-content", !online);
|
||||||
|
|
||||||
|
if (online) {
|
||||||
|
var setNum = function (sel, val) {
|
||||||
|
var el = card.querySelector(sel);
|
||||||
|
if (el) el.textContent = String(val == null ? 0 : val);
|
||||||
|
};
|
||||||
|
setNum(".site-buckets", site.buckets);
|
||||||
|
setNum(".site-objects", site.objects);
|
||||||
|
var sizeEl = card.querySelector(".site-size");
|
||||||
|
if (sizeEl) {
|
||||||
|
sizeEl.setAttribute("data-bytes", String(site.size_bytes || 0));
|
||||||
|
sizeEl.textContent = fmtBytes(site.size_bytes || 0);
|
||||||
|
}
|
||||||
|
var capacity = site.capacity || {};
|
||||||
|
var diskBar = card.querySelector(".site-disk-bar");
|
||||||
|
if (diskBar) {
|
||||||
|
diskBar.setAttribute("data-total", String(capacity.total_bytes || 0));
|
||||||
|
diskBar.setAttribute("data-available", String(capacity.available_bytes || 0));
|
||||||
|
}
|
||||||
|
var diskTotalEl = card.querySelector(".site-disk-total");
|
||||||
|
if (diskTotalEl) {
|
||||||
|
diskTotalEl.setAttribute("data-bytes", String(capacity.total_bytes || 0));
|
||||||
|
diskTotalEl.textContent = fmtBytes(capacity.total_bytes || 0);
|
||||||
|
}
|
||||||
|
var sys = site.system || {};
|
||||||
|
var setPct = function (labelSel, val) {
|
||||||
|
var label = card.querySelector(labelSel);
|
||||||
|
if (label) label.textContent = (val == null ? 0 : val) + "%";
|
||||||
|
};
|
||||||
|
setPct(".site-cpu-label", sys.cpu_percent);
|
||||||
|
setPct(".site-mem-label", sys.memory_percent);
|
||||||
|
setPct(".site-diskpct-label", sys.disk_percent);
|
||||||
|
var setBarPct = function (sel, val) {
|
||||||
|
var bar = card.querySelector(sel);
|
||||||
|
if (bar) bar.style.width = (val == null ? 0 : val) + "%";
|
||||||
|
};
|
||||||
|
setBarPct(".site-cpu-bar", sys.cpu_percent);
|
||||||
|
setBarPct(".site-mem-bar", sys.memory_percent);
|
||||||
|
setBarPct(".site-diskpct-bar", sys.disk_percent);
|
||||||
|
|
||||||
|
var sync = site.sync || {};
|
||||||
|
var syncLabel = card.querySelector(".site-sync-label");
|
||||||
|
if (syncLabel) {
|
||||||
|
if (sync.last_sync_at) {
|
||||||
|
syncLabel.innerHTML = '<span data-last-sync-at="' + sync.last_sync_at + '">last sync <span class="last-sync-rel">' + fmtRel(sync.last_sync_at) + '</span></span>';
|
||||||
|
} else {
|
||||||
|
syncLabel.innerHTML = '<span class="text-muted">no sync yet</span>';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var errBadge = card.querySelector(".site-sync-errors");
|
||||||
|
var errCount = sync.errors || 0;
|
||||||
|
if (errBadge) {
|
||||||
|
errBadge.classList.toggle("d-none", errCount === 0);
|
||||||
|
var c = errBadge.querySelector(".site-sync-errors-count");
|
||||||
|
if (c) c.textContent = errCount;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
var msg = card.querySelector(".site-offline-message");
|
||||||
|
if (msg) msg.textContent = site.error || "Peer unreachable.";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function poll(force) {
|
||||||
|
var url = "/ui/cluster/data" + (force ? "?force=1" : "");
|
||||||
|
var icon = document.getElementById("cluster-refresh-icon");
|
||||||
|
var btn = document.getElementById("cluster-refresh-btn");
|
||||||
|
if (force && icon) icon.classList.add("spin");
|
||||||
|
if (force && btn) btn.disabled = true;
|
||||||
|
return fetch(url, { credentials: "same-origin", cache: "no-store" })
|
||||||
|
.then(function (r) { return r.ok ? r.json() : null; })
|
||||||
|
.then(function (data) {
|
||||||
|
if (!data) return;
|
||||||
|
var totals = data.totals || {};
|
||||||
|
var setTotal = function (id, v) {
|
||||||
|
var el = document.getElementById(id);
|
||||||
|
if (el) el.textContent = String(v == null ? 0 : v);
|
||||||
|
};
|
||||||
|
setTotal("cluster-total-sites", totals.total_count);
|
||||||
|
setTotal("cluster-total-buckets", totals.buckets);
|
||||||
|
setTotal("cluster-total-objects", totals.objects);
|
||||||
|
var sizeEl = document.getElementById("cluster-total-size");
|
||||||
|
if (sizeEl) {
|
||||||
|
sizeEl.setAttribute("data-bytes", String(totals.size_bytes || 0));
|
||||||
|
sizeEl.textContent = fmtBytes(totals.size_bytes || 0);
|
||||||
|
}
|
||||||
|
var onlineBadge = document.getElementById("cluster-online-badge");
|
||||||
|
if (onlineBadge) onlineBadge.textContent = (totals.online_count || 0) + " / " + (totals.total_count || 0) + " online";
|
||||||
|
|
||||||
|
(data.sites || []).forEach(function (site) {
|
||||||
|
var card = document.querySelector('.site-card[data-site-id="' + site.site_id + '"]');
|
||||||
|
if (card) updateCard(card, site);
|
||||||
|
});
|
||||||
|
|
||||||
|
applyDiskBars();
|
||||||
|
applyPctBars();
|
||||||
|
var stamp = document.getElementById("cluster-updated-at");
|
||||||
|
if (stamp) stamp.textContent = "updated " + new Date().toLocaleTimeString();
|
||||||
|
})
|
||||||
|
.catch(function () { /* silent — keep last good state */ })
|
||||||
|
.finally(function () {
|
||||||
|
if (icon) icon.classList.remove("spin");
|
||||||
|
if (btn) btn.disabled = false;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
applyBytesFormat();
|
||||||
|
applyDiskBars();
|
||||||
|
applyPctBars();
|
||||||
|
refreshRel();
|
||||||
|
setInterval(refreshRel, 5000);
|
||||||
|
setInterval(function () { poll(false); }, 10000);
|
||||||
|
|
||||||
|
var refreshBtn = document.getElementById("cluster-refresh-btn");
|
||||||
|
if (refreshBtn) {
|
||||||
|
refreshBtn.addEventListener("click", function () { poll(true); });
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
</script>
|
||||||
|
<style>
|
||||||
|
@keyframes cluster-spin { from { transform: rotate(0deg); } to { transform: rotate(360deg); } }
|
||||||
|
.spin { animation: cluster-spin 0.8s linear infinite; transform-origin: 50% 50%; }
|
||||||
|
</style>
|
||||||
|
{% endblock %}
|
||||||
@@ -118,29 +118,69 @@ cargo build --release -p myfsio-server
|
|||||||
<td>Directory for buckets and objects.</td>
|
<td>Directory for buckets and objects.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><code>MAX_UPLOAD_SIZE</code></td>
|
<td><code>IAM_CONFIG</code></td>
|
||||||
<td><code>1 GB</code></td>
|
<td><code><STORAGE_ROOT>/.myfsio.sys/config/iam.json</code></td>
|
||||||
<td>Max request body size in bytes.</td>
|
<td>IAM users / access keys file path.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><code>SECRET_KEY</code></td>
|
<td><code>SECRET_KEY</code></td>
|
||||||
<td>(Auto-generated)</td>
|
<td>(loaded from <code>.myfsio.sys/config/.secret</code> when present)</td>
|
||||||
<td>Session signing and IAM-at-rest encryption key. <strong>Set explicitly in production.</strong></td>
|
<td>Session signing and IAM-at-rest encryption key. <strong>Set explicitly in production.</strong></td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><code>HOST</code></td>
|
<td><code>HOST</code></td>
|
||||||
<td><code>127.0.0.1</code></td>
|
<td><code>127.0.0.1</code></td>
|
||||||
<td>Bind interface.</td>
|
<td>Bind interface for both API and UI listeners.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><code>PORT</code></td>
|
<td><code>PORT</code></td>
|
||||||
<td><code>5000</code></td>
|
<td><code>5000</code></td>
|
||||||
<td>Listen port (UI uses 5100).</td>
|
<td>S3 API listen port.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><code>DISPLAY_TIMEZONE</code></td>
|
<td><code>UI_PORT</code></td>
|
||||||
<td><code>UTC</code></td>
|
<td><code>5100</code></td>
|
||||||
<td>Timezone for UI timestamps (e.g., <code>US/Eastern</code>, <code>Asia/Tokyo</code>).</td>
|
<td>Web UI listen port.</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>UI_ENABLED</code></td>
|
||||||
|
<td><code>true</code></td>
|
||||||
|
<td>Set to <code>false</code> to run API-only (no UI listener).</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>AWS_REGION</code></td>
|
||||||
|
<td><code>us-east-1</code></td>
|
||||||
|
<td>Region used in SigV4 scope.</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>LOG_LEVEL</code></td>
|
||||||
|
<td><code>INFO</code></td>
|
||||||
|
<td>Log verbosity (also honored via <code>RUST_LOG</code>).</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>SESSION_LIFETIME_DAYS</code></td>
|
||||||
|
<td><code>1</code></td>
|
||||||
|
<td>UI session lifetime in days.</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>REQUEST_BODY_TIMEOUT_SECONDS</code></td>
|
||||||
|
<td><code>60</code></td>
|
||||||
|
<td>Per-request body read timeout for the S3 API.</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>MULTIPART_MIN_PART_SIZE</code></td>
|
||||||
|
<td><code>5242880</code></td>
|
||||||
|
<td>Minimum part size enforced for multipart uploads (5 MiB).</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>BULK_DELETE_MAX_KEYS</code></td>
|
||||||
|
<td><code>1000</code></td>
|
||||||
|
<td>Maximum keys accepted by the UI bulk-delete endpoint.</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>STREAM_CHUNK_SIZE</code></td>
|
||||||
|
<td><code>1048576</code></td>
|
||||||
|
<td>Default streaming chunk size for routes that opt into configured chunking (1 MiB).</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr class="table-secondary">
|
<tr class="table-secondary">
|
||||||
<td colspan="3" class="fw-semibold">CORS Settings</td>
|
<td colspan="3" class="fw-semibold">CORS Settings</td>
|
||||||
@@ -166,22 +206,32 @@ cargo build --release -p myfsio-server
|
|||||||
<td>Response headers visible to browsers (e.g., <code>ETag</code>).</td>
|
<td>Response headers visible to browsers (e.g., <code>ETag</code>).</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr class="table-secondary">
|
<tr class="table-secondary">
|
||||||
<td colspan="3" class="fw-semibold">Security Settings</td>
|
<td colspan="3" class="fw-semibold">Rate Limiting</td>
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td><code>AUTH_MAX_ATTEMPTS</code></td>
|
|
||||||
<td><code>5</code></td>
|
|
||||||
<td>Failed login attempts before lockout.</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td><code>AUTH_LOCKOUT_MINUTES</code></td>
|
|
||||||
<td><code>15</code></td>
|
|
||||||
<td>Lockout duration after max failed attempts.</td>
|
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><code>RATE_LIMIT_DEFAULT</code></td>
|
<td><code>RATE_LIMIT_DEFAULT</code></td>
|
||||||
<td><code>200 per minute</code></td>
|
<td><code>5000 per minute</code></td>
|
||||||
<td>Default rate limit for S3 and KMS API endpoints.</td>
|
<td>Default rate limit for S3 and KMS API endpoints. Accepts <code>N per <second/minute/hour/day></code> or <code>N/<seconds></code>.</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>RATE_LIMIT_LIST_BUCKETS</code></td>
|
||||||
|
<td>inherits <code>RATE_LIMIT_DEFAULT</code></td>
|
||||||
|
<td>Rate limit for <code>GET /</code> (ListBuckets).</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>RATE_LIMIT_BUCKET_OPS</code></td>
|
||||||
|
<td>inherits <code>RATE_LIMIT_DEFAULT</code></td>
|
||||||
|
<td>Rate limit for bucket-scoped operations (<code>/bucket</code>).</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>RATE_LIMIT_OBJECT_OPS</code></td>
|
||||||
|
<td>inherits <code>RATE_LIMIT_DEFAULT</code></td>
|
||||||
|
<td>Rate limit for object-scoped operations (<code>/bucket/key</code>).</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>RATE_LIMIT_HEAD_OPS</code></td>
|
||||||
|
<td>inherits <code>RATE_LIMIT_DEFAULT</code></td>
|
||||||
|
<td>Rate limit applied when the request method is HEAD.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><code>RATE_LIMIT_ADMIN</code></td>
|
<td><code>RATE_LIMIT_ADMIN</code></td>
|
||||||
@@ -204,30 +254,7 @@ cargo build --release -p myfsio-server
|
|||||||
<td>Custom secret key for the admin user on first run or credential reset. Random if unset.</td>
|
<td>Custom secret key for the admin user on first run or credential reset. Random if unset.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr class="table-secondary">
|
<tr class="table-secondary">
|
||||||
<td colspan="3" class="fw-semibold">Server Settings</td>
|
<td colspan="3" class="fw-semibold">Feature Toggles</td>
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td><code>SERVER_THREADS</code></td>
|
|
||||||
<td><code>0</code> (auto)</td>
|
|
||||||
<td>Granian blocking threads (1-64). 0 = auto (CPU cores × 2).</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td><code>SERVER_CONNECTION_LIMIT</code></td>
|
|
||||||
<td><code>0</code> (auto)</td>
|
|
||||||
<td>Max concurrent connections (10-1000). 0 = auto (RAM-based).</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td><code>SERVER_BACKLOG</code></td>
|
|
||||||
<td><code>0</code> (auto)</td>
|
|
||||||
<td>TCP listen backlog (64-4096). 0 = auto (conn_limit × 2).</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td><code>SERVER_CHANNEL_TIMEOUT</code></td>
|
|
||||||
<td><code>120</code></td>
|
|
||||||
<td>Idle connection timeout in seconds (10-300).</td>
|
|
||||||
</tr>
|
|
||||||
<tr class="table-secondary">
|
|
||||||
<td colspan="3" class="fw-semibold">Encryption Settings</td>
|
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><code>ENCRYPTION_ENABLED</code></td>
|
<td><code>ENCRYPTION_ENABLED</code></td>
|
||||||
@@ -237,20 +264,32 @@ cargo build --release -p myfsio-server
|
|||||||
<tr>
|
<tr>
|
||||||
<td><code>KMS_ENABLED</code></td>
|
<td><code>KMS_ENABLED</code></td>
|
||||||
<td><code>false</code></td>
|
<td><code>false</code></td>
|
||||||
<td>Enable KMS key management for encryption.</td>
|
<td>Enable built-in KMS key management.</td>
|
||||||
</tr>
|
|
||||||
<tr class="table-secondary">
|
|
||||||
<td colspan="3" class="fw-semibold">Logging Settings</td>
|
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><code>LOG_LEVEL</code></td>
|
<td><code>GC_ENABLED</code></td>
|
||||||
<td><code>INFO</code></td>
|
<td><code>false</code></td>
|
||||||
<td>Log verbosity: DEBUG, INFO, WARNING, ERROR.</td>
|
<td>Start the garbage collector worker.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><code>LOG_TO_FILE</code></td>
|
<td><code>INTEGRITY_ENABLED</code></td>
|
||||||
<td><code>true</code></td>
|
<td><code>false</code></td>
|
||||||
<td>Enable file logging.</td>
|
<td>Start the integrity scanner worker.</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>LIFECYCLE_ENABLED</code></td>
|
||||||
|
<td><code>false</code></td>
|
||||||
|
<td>Start the lifecycle worker.</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>WEBSITE_HOSTING_ENABLED</code></td>
|
||||||
|
<td><code>false</code></td>
|
||||||
|
<td>Enable static website hosting and domain mappings.</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>SITE_SYNC_ENABLED</code></td>
|
||||||
|
<td><code>false</code></td>
|
||||||
|
<td>Start the bi-directional site sync worker.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr class="table-secondary">
|
<tr class="table-secondary">
|
||||||
<td colspan="3" class="fw-semibold">Metrics History Settings</td>
|
<td colspan="3" class="fw-semibold">Metrics History Settings</td>
|
||||||
@@ -400,16 +439,6 @@ cargo build --release -p myfsio-server
|
|||||||
<td><code>50</code></td>
|
<td><code>50</code></td>
|
||||||
<td>Max lifecycle history records per bucket.</td>
|
<td>Max lifecycle history records per bucket.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
|
||||||
<td><code>OBJECT_CACHE_TTL</code></td>
|
|
||||||
<td><code>60</code></td>
|
|
||||||
<td>Seconds to cache object metadata.</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td><code>BULK_DOWNLOAD_MAX_BYTES</code></td>
|
|
||||||
<td><code>1 GB</code></td>
|
|
||||||
<td>Max total size for bulk ZIP downloads.</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
<tr>
|
||||||
<td><code>ENCRYPTION_CHUNK_SIZE_BYTES</code></td>
|
<td><code>ENCRYPTION_CHUNK_SIZE_BYTES</code></td>
|
||||||
<td><code>65536</code></td>
|
<td><code>65536</code></td>
|
||||||
@@ -429,7 +458,7 @@ cargo build --release -p myfsio-server
|
|||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
<div class="alert alert-warning mt-3 mb-0 small">
|
<div class="alert alert-warning mt-3 mb-0 small">
|
||||||
<strong>Production Checklist:</strong> Set <code>SECRET_KEY</code> (also enables IAM config encryption at rest), restrict <code>CORS_ORIGINS</code>, configure <code>API_BASE_URL</code>, enable HTTPS via reverse proxy, use <code>--prod</code> flag, and set credential expiry on non-admin users.
|
<strong>Production Checklist:</strong> Set <code>SECRET_KEY</code> (also enables IAM config encryption at rest), restrict <code>CORS_ORIGINS</code>, configure <code>API_BASE_URL</code>, enable HTTPS via a reverse proxy, run <code>myfsio-server --check-config</code> before starting, and set credential expiry on non-admin users.
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</article>
|
</article>
|
||||||
@@ -500,7 +529,7 @@ sudo journalctl -u myfsio -f # View logs</code></pre>
|
|||||||
<div class="docs-highlight mb-3">
|
<div class="docs-highlight mb-3">
|
||||||
<ol class="mb-0">
|
<ol class="mb-0">
|
||||||
<li>Check the console output for the generated <code>Access Key</code> and <code>Secret Key</code>, then visit <code>/ui/login</code>.</li>
|
<li>Check the console output for the generated <code>Access Key</code> and <code>Secret Key</code>, then visit <code>/ui/login</code>.</li>
|
||||||
<li>Create additional users with descriptive display names, AWS-style inline policies (for example <code>{"bucket": "*", "actions": ["list", "read"]}</code>), and optional credential expiry dates.</li>
|
<li>Create additional users with descriptive display names, AWS-style inline policies (for example <code>{"bucket": "*", "actions": ["list", "read"]}</code>), and optional credential expiry dates. Use <code>{"bucket": "*", "actions": ["*"]}</code> to grant full administrator access — this is the only policy shape that satisfies <code>require_admin</code> on routes such as <code>/admin/cluster/overview</code>. <code>iam:*</code> grants only IAM-management actions and is <strong>not</strong> a substitute for <code>"*"</code> on admin routes.</li>
|
||||||
<li>Set credential expiry on users to grant time-limited access. The UI shows expiry badges and provides preset durations (1h, 24h, 7d, 30d, 90d). Expired credentials are rejected at authentication.</li>
|
<li>Set credential expiry on users to grant time-limited access. The UI shows expiry badges and provides preset durations (1h, 24h, 7d, 30d, 90d). Expired credentials are rejected at authentication.</li>
|
||||||
<li>Rotate secrets when sharing with CI jobs—new secrets display once and persist to <code>data/.myfsio.sys/config/iam.json</code>.</li>
|
<li>Rotate secrets when sharing with CI jobs—new secrets display once and persist to <code>data/.myfsio.sys/config/iam.json</code>.</li>
|
||||||
<li>Bucket policies layer on top of IAM. Apply Private/Public presets or paste custom JSON; changes reload instantly.</li>
|
<li>Bucket policies layer on top of IAM. Apply Private/Public presets or paste custom JSON; changes reload instantly.</li>
|
||||||
@@ -527,7 +556,7 @@ sudo journalctl -u myfsio -f # View logs</code></pre>
|
|||||||
<div>
|
<div>
|
||||||
<h3 class="h6 text-uppercase text-muted">Uploads</h3>
|
<h3 class="h6 text-uppercase text-muted">Uploads</h3>
|
||||||
<ul>
|
<ul>
|
||||||
<li>Drag and drop folders or files into the upload modal. Objects above 16 MB switch to multipart automatically.</li>
|
<li>Drag and drop folders or files into the upload modal. Objects above 8 MB switch to multipart automatically.</li>
|
||||||
<li>Progress rows highlight retries, throughput, and completion even if you close the modal.</li>
|
<li>Progress rows highlight retries, throughput, and completion even if you close the modal.</li>
|
||||||
</ul>
|
</ul>
|
||||||
</div>
|
</div>
|
||||||
@@ -536,7 +565,7 @@ sudo journalctl -u myfsio -f # View logs</code></pre>
|
|||||||
<ul>
|
<ul>
|
||||||
<li>Navigate folder hierarchies using breadcrumbs. Objects with <code>/</code> in keys display as folders.</li>
|
<li>Navigate folder hierarchies using breadcrumbs. Objects with <code>/</code> in keys display as folders.</li>
|
||||||
<li>Infinite scroll loads more objects automatically. Choose batch size (50–250) from the footer dropdown.</li>
|
<li>Infinite scroll loads more objects automatically. Choose batch size (50–250) from the footer dropdown.</li>
|
||||||
<li>Bulk select objects for multi-delete or multi-download (ZIP archive, up to 1 GiB). Filter by name using the search box.</li>
|
<li>Bulk select objects for multi-delete or multi-download (ZIP archive, up to 256 MB total). Filter by name using the search box.</li>
|
||||||
<li>If loading fails, click <strong>Retry</strong> to attempt again—no page refresh needed.</li>
|
<li>If loading fails, click <strong>Retry</strong> to attempt again—no page refresh needed.</li>
|
||||||
</ul>
|
</ul>
|
||||||
</div>
|
</div>
|
||||||
@@ -853,46 +882,15 @@ s3.complete_multipart_upload(
|
|||||||
</li>
|
</li>
|
||||||
</ol>
|
</ol>
|
||||||
|
|
||||||
<div class="alert alert-light border mb-3 overflow-hidden">
|
<div class="alert alert-light border mb-3">
|
||||||
<div class="d-flex flex-column flex-sm-row gap-2 mb-2">
|
<div class="d-flex gap-2">
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-terminal text-muted mt-1 flex-shrink-0 d-none d-sm-block" viewBox="0 0 16 16">
|
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-terminal text-muted mt-1 flex-shrink-0" viewBox="0 0 16 16">
|
||||||
<path d="M6 9a.5.5 0 0 1 .5-.5h3a.5.5 0 0 1 0 1h-3A.5.5 0 0 1 6 9zM3.854 4.146a.5.5 0 1 0-.708.708L4.793 6.5 3.146 8.146a.5.5 0 1 0 .708.708l2-2a.5.5 0 0 0 0-.708l-2-2z"/>
|
<path d="M6 9a.5.5 0 0 1 .5-.5h3a.5.5 0 0 1 0 1h-3A.5.5 0 0 1 6 9zM3.854 4.146a.5.5 0 1 0-.708.708L4.793 6.5 3.146 8.146a.5.5 0 1 0 .708.708l2-2a.5.5 0 0 0 0-.708l-2-2z"/>
|
||||||
<path d="M2 1a2 2 0 0 0-2 2v10a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V3a2 2 0 0 0-2-2H2zm12 1a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H2a1 1 0 0 1-1-1V3a1 1 0 0 1 1-1h12z"/>
|
<path d="M2 1a2 2 0 0 0-2 2v10a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V3a2 2 0 0 0-2-2H2zm12 1a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H2a1 1 0 0 1-1-1V3a1 1 0 0 1 1-1h12z"/>
|
||||||
</svg>
|
</svg>
|
||||||
<div class="flex-grow-1 min-width-0">
|
<div>
|
||||||
<strong>Headless Target Setup</strong>
|
<strong>Headless Target Setup</strong>
|
||||||
<p class="small text-muted mb-2">If your target server has no UI, create a <code>setup_target.py</code> script to bootstrap credentials:</p>
|
<p class="small text-muted mb-0">If your target server has no UI, start it with <code>ADMIN_ACCESS_KEY</code> and <code>ADMIN_SECRET_KEY</code> set so the first-run bootstrap installs deterministic credentials, then create the destination bucket via the AWS CLI: <code>aws --endpoint-url <target> s3api create-bucket --bucket backup-bucket</code>. Use the admin API at <code>/admin/iam/users</code> (or <code>--reset-cred</code>) to manage credentials remotely.</p>
|
||||||
<pre class="mb-0 overflow-auto" style="max-width: 100%;"><code class="language-python"># setup_target.py
|
|
||||||
from pathlib import Path
|
|
||||||
from app.iam import IamService
|
|
||||||
from app.storage import ObjectStorage
|
|
||||||
|
|
||||||
# Initialize services (paths match default config)
|
|
||||||
data_dir = Path("data")
|
|
||||||
iam = IamService(data_dir / ".myfsio.sys" / "config" / "iam.json")
|
|
||||||
storage = ObjectStorage(data_dir)
|
|
||||||
|
|
||||||
# 1. Create the bucket
|
|
||||||
bucket_name = "backup-bucket"
|
|
||||||
try:
|
|
||||||
storage.create_bucket(bucket_name)
|
|
||||||
print(f"Bucket '{bucket_name}' created.")
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Bucket creation skipped: {e}")
|
|
||||||
|
|
||||||
# 2. Create the user
|
|
||||||
try:
|
|
||||||
creds = iam.create_user(
|
|
||||||
display_name="Replication User",
|
|
||||||
policies=[{"bucket": bucket_name, "actions": ["write", "read", "list"]}]
|
|
||||||
)
|
|
||||||
print("\n--- CREDENTIALS GENERATED ---")
|
|
||||||
print(f"Access Key: {creds['access_key']}")
|
|
||||||
print(f"Secret Key: {creds['secret_key']}")
|
|
||||||
print("-----------------------------")
|
|
||||||
except Exception as e:
|
|
||||||
print(f"User creation failed: {e}")</code></pre>
|
|
||||||
<p class="small text-muted mt-2 mb-0">Save and run: <code>python setup_target.py</code></p>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -1052,15 +1050,15 @@ SITE_SYNC_BATCH_SIZE=100 # Max objects per sync cycle</code></pre>
|
|||||||
<tbody>
|
<tbody>
|
||||||
<tr>
|
<tr>
|
||||||
<td><strong>Retry Logic</strong></td>
|
<td><strong>Retry Logic</strong></td>
|
||||||
<td>boto3 automatically handles 429 (rate limit) errors using exponential backoff with <code>max_attempts=2</code></td>
|
<td>The replication worker retries failed transfers up to <code>REPLICATION_MAX_RETRIES</code> times (default <code>2</code>).</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><strong>Concurrency</strong></td>
|
<td><strong>Failure Budget</strong></td>
|
||||||
<td>Uses a ThreadPoolExecutor with 4 parallel workers for replication tasks</td>
|
<td>After <code>REPLICATION_MAX_FAILURES_PER_BUCKET</code> recorded failures (default <code>50</code>), further records for that bucket are dropped until a retry succeeds.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><strong>Timeouts</strong></td>
|
<td><strong>Timeouts</strong></td>
|
||||||
<td>Connect: 5s, Read: 30s. Large files use streaming transfers</td>
|
<td>Connect: <code>REPLICATION_CONNECT_TIMEOUT_SECONDS</code> (default 5s), Read: <code>REPLICATION_READ_TIMEOUT_SECONDS</code> (default 30s). Objects larger than <code>REPLICATION_STREAMING_THRESHOLD_BYTES</code> (default 10 MB) use streaming transfers.</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
@@ -1195,6 +1193,56 @@ cargo run -p myfsio-server --</code></pre>
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<h3 class="h6 text-uppercase text-muted mt-4">Cross-site authentication (Peer Inbound Access Key)</h3>
|
||||||
|
<p class="small text-muted">When the Cluster page asks one site to render a peer card, the local server signs a SigV4 call to <code>/admin/cluster/overview</code> on the peer. The peer needs to know which inbound caller is authorized. There are two ways for that signed call to be accepted:</p>
|
||||||
|
<ol class="small text-muted mb-3">
|
||||||
|
<li>The signing access key resolves to a real admin user on the peer (policy <code>{"bucket":"*","actions":["*"]}</code>), <strong>or</strong></li>
|
||||||
|
<li>The signing access key is whitelisted on the peer's site-registry entry as <strong>Peer Inbound Access Key</strong>. With this set, the peer accepts the call without granting full admin.</li>
|
||||||
|
</ol>
|
||||||
|
<p class="small text-muted">Option 2 is preferred — it follows least-privilege. The mental model is:</p>
|
||||||
|
<div class="alert alert-light border mb-3">
|
||||||
|
<strong class="small">Peer Inbound Access Key on Site X for peer Y</strong>
|
||||||
|
<span class="small">= "the access key Y signs with when calling X."</span>
|
||||||
|
<br>
|
||||||
|
<span class="small text-muted">It is copied from <em>the other site's outbound Connection</em>, never from your own IAM.</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h4 class="h6 mt-3 mb-2">Worked example (two sites: <code>us-east-1</code> and <code>us-west-1</code>)</h4>
|
||||||
|
<p class="small text-muted">Each site has an outbound Connection it uses to reach the other:</p>
|
||||||
|
<ul class="small text-muted mb-3">
|
||||||
|
<li><code>us-east-1</code>'s "to-west" Connection signs with access key <code>AKIA-EAST-OUT</code></li>
|
||||||
|
<li><code>us-west-1</code>'s "to-east" Connection signs with access key <code>AKIA-WEST-OUT</code></li>
|
||||||
|
</ul>
|
||||||
|
<p class="small text-muted">For the Cluster page on both sides to fetch peer data, register the peer entries with cross-paired inbound keys:</p>
|
||||||
|
<div class="row g-3 mb-3">
|
||||||
|
<div class="col-md-6">
|
||||||
|
<div class="card border h-100">
|
||||||
|
<div class="card-header bg-light py-2"><strong class="small">On us-east-1 → peer "us-west-1"</strong></div>
|
||||||
|
<div class="card-body small">
|
||||||
|
<ul class="mb-0 ps-3">
|
||||||
|
<li>Connection: "to-west"</li>
|
||||||
|
<li><strong>Peer Inbound Access Key:</strong> <code>AKIA-WEST-OUT</code></li>
|
||||||
|
<li class="text-muted">(the key us-west-1 will sign with when calling us-east-1)</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-6">
|
||||||
|
<div class="card border h-100">
|
||||||
|
<div class="card-header bg-light py-2"><strong class="small">On us-west-1 → peer "us-east-1"</strong></div>
|
||||||
|
<div class="card-body small">
|
||||||
|
<ul class="mb-0 ps-3">
|
||||||
|
<li>Connection: "to-east"</li>
|
||||||
|
<li><strong>Peer Inbound Access Key:</strong> <code>AKIA-EAST-OUT</code></li>
|
||||||
|
<li class="text-muted">(the key us-east-1 will sign with when calling us-west-1)</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<p class="small text-muted">Each side's inbound key matches the <em>other</em> side's outbound key. Putting your own admin key (e.g. <code>localadmin</code>) into your own peer entry does nothing — the inbound caller is the peer, not you.</p>
|
||||||
|
<p class="small text-muted">Each whitelisted access key still has to exist as an enabled IAM user on the receiving site (so SigV4 verification can find a matching secret). Its policy can be empty for cluster-overview alone; for site-sync to work, it needs the S3 verbs the sync workload uses (<code>list</code>, <code>read</code>, plus <code>write</code>/<code>delete</code> for inbound replication / bidirectional sync). Leave the field blank only if the signing key is a full admin on the receiving site.</p>
|
||||||
|
|
||||||
<h3 class="h6 text-uppercase text-muted mt-4">Admin API Endpoints</h3>
|
<h3 class="h6 text-uppercase text-muted mt-4">Admin API Endpoints</h3>
|
||||||
<p class="small text-muted">The <code>/admin</code> API provides programmatic access to site registry:</p>
|
<p class="small text-muted">The <code>/admin</code> API provides programmatic access to site registry:</p>
|
||||||
<pre class="mb-3"><code class="language-bash"># Get local site configuration
|
<pre class="mb-3"><code class="language-bash"># Get local site configuration
|
||||||
@@ -1353,7 +1401,7 @@ curl "{{ api_base }}/<bucket>/<key>?versionId=<version-id>" \
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<h3 class="h6 text-uppercase text-muted mt-4">Managing Quotas (Admin Only)</h3>
|
<h3 class="h6 text-uppercase text-muted mt-4">Managing Quotas (Admin Only)</h3>
|
||||||
<p class="small text-muted">Quota management is restricted to administrators (users with <code>iam:*</code> permissions).</p>
|
<p class="small text-muted">Quota management is restricted to administrators — users whose policy is <code>{"bucket": "*", "actions": ["*"]}</code>.</p>
|
||||||
<ol class="docs-steps mb-3">
|
<ol class="docs-steps mb-3">
|
||||||
<li>Navigate to your bucket → <strong>Properties</strong> tab → <strong>Storage Quota</strong> card.</li>
|
<li>Navigate to your bucket → <strong>Properties</strong> tab → <strong>Storage Quota</strong> card.</li>
|
||||||
<li>Enter limits: <strong>Max Size (MB)</strong> and/or <strong>Max Objects</strong>. Leave empty for unlimited.</li>
|
<li>Enter limits: <strong>Max Size (MB)</strong> and/or <strong>Max Objects</strong>. Leave empty for unlimited.</li>
|
||||||
@@ -1470,17 +1518,23 @@ curl -X POST {{ api_base }}/kms/keys \
|
|||||||
curl {{ api_base }}/kms/keys \
|
curl {{ api_base }}/kms/keys \
|
||||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||||||
|
|
||||||
# Rotate a key (creates new key material)
|
# Disable a key
|
||||||
curl -X POST {{ api_base }}/kms/keys/{key-id}/rotate \
|
|
||||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
|
||||||
|
|
||||||
# Disable/Enable a key
|
|
||||||
curl -X POST {{ api_base }}/kms/keys/{key-id}/disable \
|
curl -X POST {{ api_base }}/kms/keys/{key-id}/disable \
|
||||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||||||
|
|
||||||
# Schedule key deletion (30-day waiting period)
|
# Re-enable a key
|
||||||
curl -X DELETE "{{ api_base }}/kms/keys/{key-id}?waiting_period_days=30" \
|
curl -X POST {{ api_base }}/kms/keys/{key-id}/enable \
|
||||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||||||
|
|
||||||
|
# Delete a key
|
||||||
|
curl -X DELETE "{{ api_base }}/kms/keys/{key-id}" \
|
||||||
|
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||||||
|
|
||||||
|
# Generate a data key (sized between KMS_GENERATE_DATA_KEY_MIN_BYTES and _MAX_BYTES)
|
||||||
|
curl -X POST {{ api_base }}/kms/generate-data-key \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||||
|
-d '{"KeyId": "<key-id>", "NumberOfBytes": 32}'</code></pre>
|
||||||
|
|
||||||
<h3 class="h6 text-uppercase text-muted mt-4">How It Works</h3>
|
<h3 class="h6 text-uppercase text-muted mt-4">How It Works</h3>
|
||||||
<p class="small text-muted mb-0">
|
<p class="small text-muted mb-0">
|
||||||
@@ -1744,14 +1798,11 @@ curl "{{ api_base }}/admin/gc/history?limit=10" \
|
|||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
<tr><td><code>INTEGRITY_ENABLED</code></td><td><code>false</code></td><td>Enable background integrity scanning</td></tr>
|
<tr><td><code>INTEGRITY_ENABLED</code></td><td><code>false</code></td><td>Enable the background integrity scanner</td></tr>
|
||||||
<tr><td><code>INTEGRITY_INTERVAL_HOURS</code></td><td><code>24</code></td><td>Hours between scan cycles</td></tr>
|
|
||||||
<tr><td><code>INTEGRITY_BATCH_SIZE</code></td><td><code>1000</code></td><td>Max objects to scan per cycle</td></tr>
|
|
||||||
<tr><td><code>INTEGRITY_AUTO_HEAL</code></td><td><code>false</code></td><td>Automatically repair detected issues</td></tr>
|
|
||||||
<tr><td><code>INTEGRITY_DRY_RUN</code></td><td><code>false</code></td><td>Log issues without healing</td></tr>
|
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
|
<p class="small text-muted mb-3">Other scanner settings (interval, batch size, auto-heal, dry run) currently use hardcoded defaults: 24-hour interval, batch size 10 000, auto-heal off, dry run off. Use the admin API below to trigger a one-off scan with <code>auto_heal</code> or <code>dry_run</code> overrides.</p>
|
||||||
|
|
||||||
<h3 class="h6 text-uppercase text-muted mt-4">What Gets Checked</h3>
|
<h3 class="h6 text-uppercase text-muted mt-4">What Gets Checked</h3>
|
||||||
<div class="table-responsive mb-3">
|
<div class="table-responsive mb-3">
|
||||||
@@ -1819,7 +1870,7 @@ curl "{{ api_base }}/admin/integrity/history?limit=10" \
|
|||||||
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
||||||
</svg>
|
</svg>
|
||||||
<div>
|
<div>
|
||||||
<strong>Dry Run:</strong> Use <code>INTEGRITY_DRY_RUN=true</code> or pass <code>{"dry_run": true}</code> to the API to preview detected issues without making any changes. Combine with <code>{"auto_heal": true}</code> to see what would be repaired.
|
<strong>Dry Run:</strong> Pass <code>{"dry_run": true}</code> to <code>/admin/integrity/run</code> to preview detected issues without making any changes. Combine with <code>{"auto_heal": true}</code> to see what would be repaired.
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -2048,9 +2099,9 @@ curl "{{ api_base | replace(from="/api", to="/ui") }}/metrics/operations/history
|
|||||||
<td>Update IAM inline policies or remove conflicting deny statements.</td>
|
<td>Update IAM inline policies or remove conflicting deny statements.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td>Large uploads fail instantly</td>
|
<td>Large uploads time out mid-stream</td>
|
||||||
<td><code>MAX_UPLOAD_SIZE</code> exceeded</td>
|
<td><code>REQUEST_BODY_TIMEOUT_SECONDS</code> exceeded</td>
|
||||||
<td>Raise the env var or split the object.</td>
|
<td>Raise the timeout, use multipart uploads, or upload from a faster network.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td>Requests hit the wrong host</td>
|
<td>Requests hit the wrong host</td>
|
||||||
@@ -2059,8 +2110,8 @@ curl "{{ api_base | replace(from="/api", to="/ui") }}/metrics/operations/history
|
|||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td>Large folder uploads hitting rate limits (429)</td>
|
<td>Large folder uploads hitting rate limits (429)</td>
|
||||||
<td><code>RATE_LIMIT_DEFAULT</code> exceeded (200/min)</td>
|
<td><code>RATE_LIMIT_DEFAULT</code> exceeded (5000/min by default)</td>
|
||||||
<td>Increase <code>RATE_LIMIT_DEFAULT</code> in env config or upload in smaller batches. Distributed rate-limit storage is not supported yet.</td>
|
<td>Increase <code>RATE_LIMIT_DEFAULT</code> (or the per-route override), or upload in smaller batches. Distributed rate-limit storage is not supported yet.</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
@@ -2079,7 +2130,7 @@ curl "{{ api_base | replace(from="/api", to="/ui") }}/metrics/operations/history
|
|||||||
curl {{ api_base }}/myfsio/health
|
curl {{ api_base }}/myfsio/health
|
||||||
|
|
||||||
# Response
|
# Response
|
||||||
{"status": "ok", "version": "0.1.7"}</code></pre>
|
{"status": "ok", "version": "0.5.0"}</code></pre>
|
||||||
|
|
||||||
<p class="small text-muted mb-3">Use this endpoint for:</p>
|
<p class="small text-muted mb-3">Use this endpoint for:</p>
|
||||||
<ul class="small text-muted mb-0">
|
<ul class="small text-muted mb-0">
|
||||||
@@ -2248,7 +2299,7 @@ curl -X PUT "{{ api_base }}/<bucket>?notification" \
|
|||||||
<p class="text-muted">Query CSV, JSON, or Parquet files directly using SQL without downloading the entire object.</p>
|
<p class="text-muted">Query CSV, JSON, or Parquet files directly using SQL without downloading the entire object.</p>
|
||||||
|
|
||||||
<div class="alert alert-info border small mb-3">
|
<div class="alert alert-info border small mb-3">
|
||||||
<strong>Prerequisite:</strong> Requires DuckDB to be installed (<code>pip install duckdb</code>)
|
<strong>Note:</strong> DuckDB is bundled into the Rust server binary — no separate install is required.
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<pre class="mb-3"><code class="language-bash"># Query a CSV file
|
<pre class="mb-3"><code class="language-bash"># Query a CSV file
|
||||||
@@ -2816,20 +2867,36 @@ GET|PUT /admin/site # Local site config
|
|||||||
GET /admin/sites # List peers
|
GET /admin/sites # List peers
|
||||||
POST /admin/sites # Register peer
|
POST /admin/sites # Register peer
|
||||||
GET|PUT|DELETE /admin/sites/<id> # Manage peer
|
GET|PUT|DELETE /admin/sites/<id> # Manage peer
|
||||||
GET /admin/sites/<id>/health # Peer health
|
GET|POST /admin/sites/<id>/health # Peer health
|
||||||
|
GET /admin/sites/<id>/bidirectional-status # Bidi sync state
|
||||||
GET /admin/topology # Cluster topology
|
GET /admin/topology # Cluster topology
|
||||||
GET|POST|PUT|DELETE /admin/website-domains # Domain mappings
|
GET|POST /admin/website-domains # List / Create domain mapping
|
||||||
|
GET|PUT|DELETE /admin/website-domains/<domain> # Manage domain mapping
|
||||||
|
GET /admin/iam/users # List IAM users
|
||||||
|
GET /admin/iam/users/<id> # Get user
|
||||||
|
GET /admin/iam/users/<id>/policies # Get user policies
|
||||||
|
POST /admin/iam/users/<id>/access-keys # Create access key
|
||||||
|
DELETE /admin/iam/users/<id>/access-keys/<ak> # Delete access key
|
||||||
|
POST /admin/iam/users/<id>/disable # Disable user
|
||||||
|
POST /admin/iam/users/<id>/enable # Enable user
|
||||||
|
GET|POST /admin/gc/status, /admin/gc/run, /admin/gc/history
|
||||||
|
GET|POST /admin/integrity/status, /admin/integrity/run, /admin/integrity/history
|
||||||
|
|
||||||
# KMS API
|
# KMS API (only mounted when KMS_ENABLED=true)
|
||||||
GET|POST /kms/keys # List / Create keys
|
GET|POST /kms/keys # List / Create keys
|
||||||
GET|DELETE /kms/keys/<id> # Get / Delete key
|
GET|DELETE /kms/keys/<id> # Get / Delete key
|
||||||
POST /kms/keys/<id>/enable # Enable key
|
POST /kms/keys/<id>/enable # Enable key
|
||||||
POST /kms/keys/<id>/disable # Disable key
|
POST /kms/keys/<id>/disable # Disable key
|
||||||
POST /kms/keys/<id>/rotate # Rotate key
|
POST /kms/encrypt # Encrypt data
|
||||||
POST /kms/encrypt # Encrypt data
|
POST /kms/decrypt # Decrypt data
|
||||||
POST /kms/decrypt # Decrypt data
|
POST /kms/re-encrypt # Re-encrypt under a different key
|
||||||
POST /kms/generate-data-key # Generate data key
|
POST /kms/generate-data-key # Generate data key
|
||||||
POST /kms/generate-random # Generate random bytes</code></pre>
|
POST /kms/generate-data-key-without-plaintext # Generate wrapped DEK only
|
||||||
|
POST /kms/generate-random # Generate random bytes
|
||||||
|
POST /kms/client/generate-key # Client-side key helper
|
||||||
|
POST /kms/client/encrypt # Client-side encrypt helper
|
||||||
|
POST /kms/client/decrypt # Client-side decrypt helper
|
||||||
|
POST /kms/materials/<id> # Fetch wrapped key materials</code></pre>
|
||||||
</div>
|
</div>
|
||||||
</article>
|
</article>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -31,7 +31,7 @@
|
|||||||
{% if iam_locked %}
|
{% if iam_locked %}
|
||||||
<div class="alert alert-warning" role="alert">
|
<div class="alert alert-warning" role="alert">
|
||||||
<div class="fw-semibold mb-1">Administrator permissions required</div>
|
<div class="fw-semibold mb-1">Administrator permissions required</div>
|
||||||
<p class="mb-0">You need the <code>iam:list_users</code> action to edit users or policies. {{ locked_reason or "Sign in with an admin identity to continue." }}</p>
|
<p class="mb-0">You need the <code>iam:list_users</code> action to edit users or policies. {% if locked_reason %}{{ locked_reason }}{% else %}Sign in with an admin identity to continue.{% endif %}</p>
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@
|
|||||||
</svg>
|
</svg>
|
||||||
Set Up Replication
|
Set Up Replication
|
||||||
</h1>
|
</h1>
|
||||||
<p class="text-muted mb-0 mt-1">Configure bucket replication to <strong>{{ peer.display_name or peer.site_id }}</strong></p>
|
<p class="text-muted mb-0 mt-1">Configure bucket replication to <strong>{% if peer.display_name %}{{ peer.display_name }}{% else %}{{ peer.site_id }}{% endif %}</strong></p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -100,7 +100,7 @@
|
|||||||
<hr class="my-2">
|
<hr class="my-2">
|
||||||
<p class="mb-2 fw-semibold">After completing this wizard, you must also:</p>
|
<p class="mb-2 fw-semibold">After completing this wizard, you must also:</p>
|
||||||
<ol class="mb-2 ps-3">
|
<ol class="mb-2 ps-3">
|
||||||
<li>Go to <strong>{{ peer.display_name or peer.site_id }}</strong>'s admin UI</li>
|
<li>Go to <strong>{% if peer.display_name %}{{ peer.display_name }}{% else %}{{ peer.site_id }}{% endif %}</strong>'s admin UI</li>
|
||||||
<li>Register <strong>this site</strong> as a peer (with a connection)</li>
|
<li>Register <strong>this site</strong> as a peer (with a connection)</li>
|
||||||
<li>Create matching bidirectional replication rules pointing back to this site</li>
|
<li>Create matching bidirectional replication rules pointing back to this site</li>
|
||||||
<li>Ensure <code>SITE_SYNC_ENABLED=true</code> is set on both sites</li>
|
<li>Ensure <code>SITE_SYNC_ENABLED=true</code> is set on both sites</li>
|
||||||
@@ -147,7 +147,7 @@
|
|||||||
<td>
|
<td>
|
||||||
<input type="text" class="form-control form-control-sm"
|
<input type="text" class="form-control form-control-sm"
|
||||||
name="target_{{ bucket.name }}"
|
name="target_{{ bucket.name }}"
|
||||||
value="{{ bucket.existing_target or bucket.name }}"
|
value="{% if bucket.existing_target %}{{ bucket.existing_target }}{% else %}{{ bucket.name }}{% endif %}"
|
||||||
placeholder="{{ bucket.name }}"
|
placeholder="{{ bucket.name }}"
|
||||||
{% if bucket.has_rule %}disabled{% endif %}>
|
{% if bucket.has_rule %}disabled{% endif %}>
|
||||||
</td>
|
</td>
|
||||||
|
|||||||
@@ -142,6 +142,11 @@
|
|||||||
</select>
|
</select>
|
||||||
<div class="form-text">Link to a remote connection for health checks</div>
|
<div class="form-text">Link to a remote connection for health checks</div>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="mb-3">
|
||||||
|
<label for="peer_inbound_access_key" class="form-label fw-medium">Peer Inbound Access Key</label>
|
||||||
|
<input type="text" class="form-control" id="peer_inbound_access_key" name="peer_inbound_access_key" placeholder="AKIA... (optional)" autocomplete="off" spellcheck="false">
|
||||||
|
<div class="form-text">Access key the peer presents when calling this site (e.g. /admin/cluster/overview). Leave blank to require admin credentials.</div>
|
||||||
|
</div>
|
||||||
<div class="d-grid">
|
<div class="d-grid">
|
||||||
<button type="submit" class="btn btn-primary">
|
<button type="submit" class="btn btn-primary">
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||||
@@ -225,7 +230,7 @@
|
|||||||
</svg>
|
</svg>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<span class="fw-medium">{{ peer.display_name or peer.site_id }}</span>
|
<span class="fw-medium">{% if peer.display_name %}{{ peer.display_name }}{% else %}{{ peer.site_id }}{% endif %}</span>
|
||||||
{% if peer.display_name and peer.display_name != peer.site_id %}
|
{% if peer.display_name and peer.display_name != peer.site_id %}
|
||||||
<br><small class="text-muted">{{ peer.site_id }}</small>
|
<br><small class="text-muted">{{ peer.site_id }}</small>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
@@ -257,7 +262,16 @@
|
|||||||
</svg>
|
</svg>
|
||||||
</span>
|
</span>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
{% if item.errors and item.errors > 0 %}
|
||||||
|
<span class="badge bg-danger bg-opacity-10 text-danger" title="Sync errors across bidirectional buckets">{{ item.errors }} err</span>
|
||||||
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
|
{% if item.last_sync_at %}
|
||||||
|
<div class="text-muted small mt-1" data-last-sync-at="{{ item.last_sync_at }}">
|
||||||
|
last sync: <span class="last-sync-rel">just now</span>
|
||||||
|
{% if item.objects_pulled %} · {{ item.objects_pulled }} pulled{% endif %}
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
<div class="sync-stats-detail d-none mt-2 small" id="stats-{{ peer.site_id }}">
|
<div class="sync-stats-detail d-none mt-2 small" id="stats-{{ peer.site_id }}">
|
||||||
<span class="spinner-border spinner-border-sm text-muted" style="width: 12px; height: 12px;"></span>
|
<span class="spinner-border spinner-border-sm text-muted" style="width: 12px; height: 12px;"></span>
|
||||||
</div>
|
</div>
|
||||||
@@ -278,6 +292,7 @@
|
|||||||
data-priority="{{ peer.priority }}"
|
data-priority="{{ peer.priority }}"
|
||||||
data-display-name="{{ peer.display_name }}"
|
data-display-name="{{ peer.display_name }}"
|
||||||
data-connection-id="{{ peer.connection_id | default(value="") }}"
|
data-connection-id="{{ peer.connection_id | default(value="") }}"
|
||||||
|
data-peer-inbound-access-key="{{ peer.peer_inbound_access_key | default(value="") }}"
|
||||||
title="Edit peer">
|
title="Edit peer">
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||||
<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/>
|
<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/>
|
||||||
@@ -301,7 +316,7 @@
|
|||||||
<li>
|
<li>
|
||||||
<button type="button" class="dropdown-item btn-check-bidir {% if not item.has_connection %}disabled{% endif %}"
|
<button type="button" class="dropdown-item btn-check-bidir {% if not item.has_connection %}disabled{% endif %}"
|
||||||
data-site-id="{{ peer.site_id }}"
|
data-site-id="{{ peer.site_id }}"
|
||||||
data-display-name="{{ peer.display_name or peer.site_id }}">
|
data-display-name="{% if peer.display_name %}{{ peer.display_name }}{% else %}{{ peer.site_id }}{% endif %}">
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2 text-info" viewBox="0 0 16 16">
|
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2 text-info" viewBox="0 0 16 16">
|
||||||
<path fill-rule="evenodd" d="M1 11.5a.5.5 0 0 0 .5.5h11.793l-3.147 3.146a.5.5 0 0 0 .708.708l4-4a.5.5 0 0 0 0-.708l-4-4a.5.5 0 0 0-.708.708L13.293 11H1.5a.5.5 0 0 0-.5.5zm14-7a.5.5 0 0 1-.5.5H2.707l3.147 3.146a.5.5 0 1 1-.708.708l-4-4a.5.5 0 0 1 0-.708l4-4a.5.5 0 1 1 .708.708L2.707 4H14.5a.5.5 0 0 1 .5.5z"/>
|
<path fill-rule="evenodd" d="M1 11.5a.5.5 0 0 0 .5.5h11.793l-3.147 3.146a.5.5 0 0 0 .708.708l4-4a.5.5 0 0 0 0-.708l-4-4a.5.5 0 0 0-.708.708L13.293 11H1.5a.5.5 0 0 0-.5.5zm14-7a.5.5 0 0 1-.5.5H2.707l3.147 3.146a.5.5 0 1 1-.708.708l-4-4a.5.5 0 0 1 0-.708l4-4a.5.5 0 1 1 .708.708L2.707 4H14.5a.5.5 0 0 1 .5.5z"/>
|
||||||
</svg>
|
</svg>
|
||||||
@@ -335,7 +350,7 @@
|
|||||||
data-bs-toggle="modal"
|
data-bs-toggle="modal"
|
||||||
data-bs-target="#deletePeerModal"
|
data-bs-target="#deletePeerModal"
|
||||||
data-site-id="{{ peer.site_id }}"
|
data-site-id="{{ peer.site_id }}"
|
||||||
data-display-name="{{ peer.display_name or peer.site_id }}">
|
data-display-name="{% if peer.display_name %}{{ peer.display_name }}{% else %}{{ peer.site_id }}{% endif %}">
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16">
|
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16">
|
||||||
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
|
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
|
||||||
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
|
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
|
||||||
@@ -385,7 +400,7 @@
|
|||||||
<div class="modal-body">
|
<div class="modal-body">
|
||||||
<div class="mb-3">
|
<div class="mb-3">
|
||||||
<label class="form-label fw-medium">Site ID</label>
|
<label class="form-label fw-medium">Site ID</label>
|
||||||
<input type="text" class="form-control" id="edit_site_id" readonly>
|
<input type="text" class="form-control" id="edit_site_id" name="site_id" readonly>
|
||||||
</div>
|
</div>
|
||||||
<div class="mb-3">
|
<div class="mb-3">
|
||||||
<label for="edit_endpoint" class="form-label fw-medium">Endpoint URL</label>
|
<label for="edit_endpoint" class="form-label fw-medium">Endpoint URL</label>
|
||||||
@@ -414,6 +429,11 @@
|
|||||||
{% endfor %}
|
{% endfor %}
|
||||||
</select>
|
</select>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="mb-3">
|
||||||
|
<label for="edit_peer_inbound_access_key" class="form-label fw-medium">Peer Inbound Access Key</label>
|
||||||
|
<input type="text" class="form-control" id="edit_peer_inbound_access_key" name="peer_inbound_access_key" placeholder="AKIA... (optional)" autocomplete="off" spellcheck="false">
|
||||||
|
<div class="form-text">Access key the peer presents when calling this site (e.g. /admin/cluster/overview). Leave blank to require admin credentials.</div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="modal-footer">
|
<div class="modal-footer">
|
||||||
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
|
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||||
@@ -516,6 +536,7 @@
|
|||||||
document.getElementById('edit_priority').value = button.getAttribute('data-priority');
|
document.getElementById('edit_priority').value = button.getAttribute('data-priority');
|
||||||
document.getElementById('edit_display_name').value = button.getAttribute('data-display-name');
|
document.getElementById('edit_display_name').value = button.getAttribute('data-display-name');
|
||||||
document.getElementById('edit_connection_id').value = button.getAttribute('data-connection-id');
|
document.getElementById('edit_connection_id').value = button.getAttribute('data-connection-id');
|
||||||
|
document.getElementById('edit_peer_inbound_access_key').value = button.getAttribute('data-peer-inbound-access-key') || '';
|
||||||
document.getElementById('editPeerForm').action = '/ui/sites/peers/' + encodeURIComponent(siteId) + '/update';
|
document.getElementById('editPeerForm').action = '/ui/sites/peers/' + encodeURIComponent(siteId) + '/update';
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -859,17 +880,64 @@
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
document.querySelectorAll('.peer-actions-dropdown').forEach(function(dd) {
|
document.querySelectorAll('.peer-actions-dropdown').forEach(function(dd, idx) {
|
||||||
dd.addEventListener('shown.bs.dropdown', function() {
|
var menu = dd.querySelector('.dropdown-menu');
|
||||||
|
if (!menu) return;
|
||||||
|
var pairId = 'peer-dd-' + idx;
|
||||||
|
dd.dataset.peerDdPair = pairId;
|
||||||
|
menu.dataset.peerDdPair = pairId;
|
||||||
|
function reposition() {
|
||||||
var toggle = dd.querySelector('[data-bs-toggle="dropdown"]');
|
var toggle = dd.querySelector('[data-bs-toggle="dropdown"]');
|
||||||
var menu = dd.querySelector('.dropdown-menu');
|
if (!toggle) return;
|
||||||
if (!toggle || !menu) return;
|
|
||||||
var rect = toggle.getBoundingClientRect();
|
var rect = toggle.getBoundingClientRect();
|
||||||
menu.style.top = rect.bottom + 'px';
|
var menuWidth = menu.offsetWidth;
|
||||||
menu.style.left = (rect.right - menu.offsetWidth) + 'px';
|
var menuHeight = menu.offsetHeight;
|
||||||
|
var pad = 8;
|
||||||
|
var left = rect.right - menuWidth;
|
||||||
|
if (left + menuWidth > window.innerWidth - pad) left = window.innerWidth - pad - menuWidth;
|
||||||
|
if (left < pad) left = pad;
|
||||||
|
var top = rect.bottom;
|
||||||
|
if (top + menuHeight > window.innerHeight - pad) {
|
||||||
|
top = Math.max(pad, rect.top - menuHeight);
|
||||||
|
}
|
||||||
|
menu.style.position = 'fixed';
|
||||||
|
menu.style.top = top + 'px';
|
||||||
|
menu.style.left = left + 'px';
|
||||||
|
menu.style.right = 'auto';
|
||||||
|
menu.style.bottom = 'auto';
|
||||||
|
menu.style.transform = 'none';
|
||||||
|
}
|
||||||
|
dd.addEventListener('show.bs.dropdown', function() {
|
||||||
|
if (menu.parentNode !== document.body) document.body.appendChild(menu);
|
||||||
});
|
});
|
||||||
|
dd.addEventListener('shown.bs.dropdown', reposition);
|
||||||
|
dd.addEventListener('hidden.bs.dropdown', function() {
|
||||||
|
menu.style.cssText = '';
|
||||||
|
if (menu.parentNode !== dd) dd.appendChild(menu);
|
||||||
|
});
|
||||||
|
window.addEventListener('resize', function() { if (menu.classList.contains('show')) reposition(); });
|
||||||
|
window.addEventListener('scroll', function() { if (menu.classList.contains('show')) reposition(); }, true);
|
||||||
});
|
});
|
||||||
})();
|
})();
|
||||||
|
|
||||||
|
(function () {
|
||||||
|
function fmtRel(ts) {
|
||||||
|
var diff = Math.max(0, Math.floor(Date.now() / 1000 - ts));
|
||||||
|
if (diff < 60) return diff + "s ago";
|
||||||
|
if (diff < 3600) return Math.floor(diff / 60) + "m ago";
|
||||||
|
if (diff < 86400) return Math.floor(diff / 3600) + "h ago";
|
||||||
|
return Math.floor(diff / 86400) + "d ago";
|
||||||
|
}
|
||||||
|
function refresh() {
|
||||||
|
document.querySelectorAll("[data-last-sync-at]").forEach(function (el) {
|
||||||
|
var ts = parseFloat(el.getAttribute("data-last-sync-at"));
|
||||||
|
var span = el.querySelector(".last-sync-rel");
|
||||||
|
if (span && !isNaN(ts)) span.textContent = fmtRel(ts);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
refresh();
|
||||||
|
setInterval(refresh, 30000);
|
||||||
|
})();
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style>
|
<style>
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -10,7 +10,7 @@ fn engine() -> TemplateEngine {
|
|||||||
path.push("templates");
|
path.push("templates");
|
||||||
path.push("*.html");
|
path.push("*.html");
|
||||||
let glob = path.to_string_lossy().replace('\\', "/");
|
let glob = path.to_string_lossy().replace('\\', "/");
|
||||||
let engine = TemplateEngine::new(&glob).expect("template parse");
|
let engine = TemplateEngine::new(&glob, "UTC").expect("template parse");
|
||||||
myfsio_server::handlers::ui_pages::register_ui_endpoints(&engine);
|
myfsio_server::handlers::ui_pages::register_ui_endpoints(&engine);
|
||||||
engine
|
engine
|
||||||
}
|
}
|
||||||
@@ -219,6 +219,59 @@ fn render_sites() {
|
|||||||
render_or_panic("sites.html", &ctx);
|
render_or_panic("sites.html", &ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn render_cluster_empty() {
|
||||||
|
let mut ctx = base_ctx();
|
||||||
|
ctx.insert("cluster_sites", &Vec::<Value>::new());
|
||||||
|
ctx.insert("cluster_total_buckets", &0u64);
|
||||||
|
ctx.insert("cluster_total_objects", &0u64);
|
||||||
|
ctx.insert("cluster_total_size_bytes", &0u64);
|
||||||
|
ctx.insert("cluster_online_count", &0usize);
|
||||||
|
ctx.insert("cluster_total_count", &0usize);
|
||||||
|
render_or_panic("cluster.html", &ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn render_cluster_with_sites() {
|
||||||
|
let mut ctx = base_ctx();
|
||||||
|
let sites = json!([
|
||||||
|
{
|
||||||
|
"site_id": "local-1",
|
||||||
|
"display_name": "Local",
|
||||||
|
"endpoint": "http://127.0.0.1:8000",
|
||||||
|
"region": "us-east-1",
|
||||||
|
"online": true,
|
||||||
|
"stale": false,
|
||||||
|
"is_local": true,
|
||||||
|
"buckets": 3,
|
||||||
|
"objects": 42,
|
||||||
|
"size_bytes": 1048576,
|
||||||
|
"capacity": {"total_bytes": 100000000, "available_bytes": 50000000},
|
||||||
|
"system": {"cpu_percent": 12.5, "memory_percent": 33.0, "disk_percent": 50.0, "storage_bytes": 1048576},
|
||||||
|
"sync": {"errors": 0, "last_sync_at": 1700000000.0},
|
||||||
|
"error": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"site_id": "peer-1",
|
||||||
|
"display_name": "Peer",
|
||||||
|
"endpoint": "http://peer.example.com",
|
||||||
|
"online": false,
|
||||||
|
"stale": true,
|
||||||
|
"is_local": false,
|
||||||
|
"registered_region": "us-west-2",
|
||||||
|
"registered_priority": 100,
|
||||||
|
"error": "request failed: timeout"
|
||||||
|
}
|
||||||
|
]);
|
||||||
|
ctx.insert("cluster_sites", &sites);
|
||||||
|
ctx.insert("cluster_total_buckets", &3u64);
|
||||||
|
ctx.insert("cluster_total_objects", &42u64);
|
||||||
|
ctx.insert("cluster_total_size_bytes", &1048576u64);
|
||||||
|
ctx.insert("cluster_online_count", &1usize);
|
||||||
|
ctx.insert("cluster_total_count", &2usize);
|
||||||
|
render_or_panic("cluster.html", &ctx);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn render_website_domains() {
|
fn render_website_domains() {
|
||||||
let mut ctx = base_ctx();
|
let mut ctx = base_ctx();
|
||||||
@@ -288,6 +341,7 @@ fn render_bucket_detail() {
|
|||||||
ctx.insert("bytes_pct", &0);
|
ctx.insert("bytes_pct", &0);
|
||||||
ctx.insert("has_quota", &false);
|
ctx.insert("has_quota", &false);
|
||||||
ctx.insert("versioning_enabled", &false);
|
ctx.insert("versioning_enabled", &false);
|
||||||
|
ctx.insert("versioning_suspended", &false);
|
||||||
ctx.insert("versioning_status", &"Disabled");
|
ctx.insert("versioning_status", &"Disabled");
|
||||||
ctx.insert("encryption_config", &json!({"Rules": []}));
|
ctx.insert("encryption_config", &json!({"Rules": []}));
|
||||||
ctx.insert("enc_rules", &Vec::<Value>::new());
|
ctx.insert("enc_rules", &Vec::<Value>::new());
|
||||||
@@ -369,6 +423,7 @@ fn render_bucket_detail_without_error_document() {
|
|||||||
ctx.insert("bytes_pct", &0);
|
ctx.insert("bytes_pct", &0);
|
||||||
ctx.insert("has_quota", &false);
|
ctx.insert("has_quota", &false);
|
||||||
ctx.insert("versioning_enabled", &false);
|
ctx.insert("versioning_enabled", &false);
|
||||||
|
ctx.insert("versioning_suspended", &false);
|
||||||
ctx.insert("versioning_status", &"Disabled");
|
ctx.insert("versioning_status", &"Disabled");
|
||||||
ctx.insert("encryption_config", &json!({"Rules": []}));
|
ctx.insert("encryption_config", &json!({"Rules": []}));
|
||||||
ctx.insert("enc_rules", &Vec::<Value>::new());
|
ctx.insert("enc_rules", &Vec::<Value>::new());
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ myfsio-crypto = { path = "../myfsio-crypto" }
|
|||||||
serde = { workspace = true }
|
serde = { workspace = true }
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
tokio = { workspace = true }
|
tokio = { workspace = true }
|
||||||
|
tokio-util = { workspace = true }
|
||||||
dashmap = { workspace = true }
|
dashmap = { workspace = true }
|
||||||
parking_lot = { workspace = true }
|
parking_lot = { workspace = true }
|
||||||
uuid = { workspace = true }
|
uuid = { workspace = true }
|
||||||
|
|||||||
@@ -17,10 +17,24 @@ pub enum StorageError {
|
|||||||
key: String,
|
key: String,
|
||||||
version_id: String,
|
version_id: String,
|
||||||
},
|
},
|
||||||
|
#[error("Object is a delete marker: {bucket}/{key}")]
|
||||||
|
DeleteMarker {
|
||||||
|
bucket: String,
|
||||||
|
key: String,
|
||||||
|
version_id: String,
|
||||||
|
},
|
||||||
|
#[error("Object corrupted: {bucket}/{key} ({detail})")]
|
||||||
|
ObjectCorrupted {
|
||||||
|
bucket: String,
|
||||||
|
key: String,
|
||||||
|
detail: String,
|
||||||
|
},
|
||||||
#[error("Invalid bucket name: {0}")]
|
#[error("Invalid bucket name: {0}")]
|
||||||
InvalidBucketName(String),
|
InvalidBucketName(String),
|
||||||
#[error("Invalid object key: {0}")]
|
#[error("Invalid object key: {0}")]
|
||||||
InvalidObjectKey(String),
|
InvalidObjectKey(String),
|
||||||
|
#[error("Method not allowed: {0}")]
|
||||||
|
MethodNotAllowed(String),
|
||||||
#[error("Upload not found: {0}")]
|
#[error("Upload not found: {0}")]
|
||||||
UploadNotFound(String),
|
UploadNotFound(String),
|
||||||
#[error("Quota exceeded: {0}")]
|
#[error("Quota exceeded: {0}")]
|
||||||
@@ -42,7 +56,7 @@ impl From<StorageError> for S3Error {
|
|||||||
S3Error::from_code(S3ErrorCode::NoSuchBucket).with_resource(format!("/{}", name))
|
S3Error::from_code(S3ErrorCode::NoSuchBucket).with_resource(format!("/{}", name))
|
||||||
}
|
}
|
||||||
StorageError::BucketAlreadyExists(name) => {
|
StorageError::BucketAlreadyExists(name) => {
|
||||||
S3Error::from_code(S3ErrorCode::BucketAlreadyExists)
|
S3Error::from_code(S3ErrorCode::BucketAlreadyOwnedByYou)
|
||||||
.with_resource(format!("/{}", name))
|
.with_resource(format!("/{}", name))
|
||||||
}
|
}
|
||||||
StorageError::BucketNotEmpty(name) => {
|
StorageError::BucketNotEmpty(name) => {
|
||||||
@@ -58,10 +72,26 @@ impl From<StorageError> for S3Error {
|
|||||||
version_id,
|
version_id,
|
||||||
} => S3Error::from_code(S3ErrorCode::NoSuchVersion)
|
} => S3Error::from_code(S3ErrorCode::NoSuchVersion)
|
||||||
.with_resource(format!("/{}/{}?versionId={}", bucket, key, version_id)),
|
.with_resource(format!("/{}/{}?versionId={}", bucket, key, version_id)),
|
||||||
|
StorageError::DeleteMarker {
|
||||||
|
bucket,
|
||||||
|
key,
|
||||||
|
version_id,
|
||||||
|
} => S3Error::from_code(S3ErrorCode::MethodNotAllowed)
|
||||||
|
.with_resource(format!("/{}/{}?versionId={}", bucket, key, version_id)),
|
||||||
|
StorageError::ObjectCorrupted {
|
||||||
|
bucket,
|
||||||
|
key,
|
||||||
|
detail,
|
||||||
|
} => S3Error::new(
|
||||||
|
S3ErrorCode::ObjectCorrupted,
|
||||||
|
format!("Object corrupted: {}", detail),
|
||||||
|
)
|
||||||
|
.with_resource(format!("/{}/{}", bucket, key)),
|
||||||
StorageError::InvalidBucketName(msg) => {
|
StorageError::InvalidBucketName(msg) => {
|
||||||
S3Error::new(S3ErrorCode::InvalidBucketName, msg)
|
S3Error::new(S3ErrorCode::InvalidBucketName, msg)
|
||||||
}
|
}
|
||||||
StorageError::InvalidObjectKey(msg) => S3Error::new(S3ErrorCode::InvalidKey, msg),
|
StorageError::InvalidObjectKey(msg) => S3Error::new(S3ErrorCode::InvalidKey, msg),
|
||||||
|
StorageError::MethodNotAllowed(msg) => S3Error::new(S3ErrorCode::MethodNotAllowed, msg),
|
||||||
StorageError::UploadNotFound(id) => S3Error::new(
|
StorageError::UploadNotFound(id) => S3Error::new(
|
||||||
S3ErrorCode::NoSuchUpload,
|
S3ErrorCode::NoSuchUpload,
|
||||||
format!("Upload {} not found", id),
|
format!("Upload {} not found", id),
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -30,8 +30,44 @@ pub trait StorageEngine: Send + Sync {
|
|||||||
key: &str,
|
key: &str,
|
||||||
) -> StorageResult<(ObjectMeta, AsyncReadStream)>;
|
) -> StorageResult<(ObjectMeta, AsyncReadStream)>;
|
||||||
|
|
||||||
|
async fn get_object_range(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
start: u64,
|
||||||
|
len: Option<u64>,
|
||||||
|
) -> StorageResult<(ObjectMeta, AsyncReadStream)>;
|
||||||
|
|
||||||
|
async fn get_object_snapshot(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
) -> StorageResult<(ObjectMeta, tokio::fs::File)>;
|
||||||
|
|
||||||
|
async fn get_object_version_snapshot(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
) -> StorageResult<(ObjectMeta, tokio::fs::File)>;
|
||||||
|
|
||||||
async fn get_object_path(&self, bucket: &str, key: &str) -> StorageResult<PathBuf>;
|
async fn get_object_path(&self, bucket: &str, key: &str) -> StorageResult<PathBuf>;
|
||||||
|
|
||||||
|
async fn snapshot_object_to_link(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
link_path: &std::path::Path,
|
||||||
|
) -> StorageResult<ObjectMeta>;
|
||||||
|
|
||||||
|
async fn snapshot_object_version_to_link(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
link_path: &std::path::Path,
|
||||||
|
) -> StorageResult<ObjectMeta>;
|
||||||
|
|
||||||
async fn head_object(&self, bucket: &str, key: &str) -> StorageResult<ObjectMeta>;
|
async fn head_object(&self, bucket: &str, key: &str) -> StorageResult<ObjectMeta>;
|
||||||
|
|
||||||
async fn get_object_version(
|
async fn get_object_version(
|
||||||
@@ -41,6 +77,15 @@ pub trait StorageEngine: Send + Sync {
|
|||||||
version_id: &str,
|
version_id: &str,
|
||||||
) -> StorageResult<(ObjectMeta, AsyncReadStream)>;
|
) -> StorageResult<(ObjectMeta, AsyncReadStream)>;
|
||||||
|
|
||||||
|
async fn get_object_version_range(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
start: u64,
|
||||||
|
len: Option<u64>,
|
||||||
|
) -> StorageResult<(ObjectMeta, AsyncReadStream)>;
|
||||||
|
|
||||||
async fn get_object_version_path(
|
async fn get_object_version_path(
|
||||||
&self,
|
&self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -62,14 +107,20 @@ pub trait StorageEngine: Send + Sync {
|
|||||||
version_id: &str,
|
version_id: &str,
|
||||||
) -> StorageResult<HashMap<String, String>>;
|
) -> StorageResult<HashMap<String, String>>;
|
||||||
|
|
||||||
async fn delete_object(&self, bucket: &str, key: &str) -> StorageResult<()>;
|
async fn get_archived_null_version_metadata(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
) -> StorageResult<Option<HashMap<String, String>>>;
|
||||||
|
|
||||||
|
async fn delete_object(&self, bucket: &str, key: &str) -> StorageResult<DeleteOutcome>;
|
||||||
|
|
||||||
async fn delete_object_version(
|
async fn delete_object_version(
|
||||||
&self,
|
&self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
key: &str,
|
key: &str,
|
||||||
version_id: &str,
|
version_id: &str,
|
||||||
) -> StorageResult<()>;
|
) -> StorageResult<DeleteOutcome>;
|
||||||
|
|
||||||
async fn copy_object(
|
async fn copy_object(
|
||||||
&self,
|
&self,
|
||||||
@@ -126,6 +177,7 @@ pub trait StorageEngine: Send + Sync {
|
|||||||
part_number: u32,
|
part_number: u32,
|
||||||
src_bucket: &str,
|
src_bucket: &str,
|
||||||
src_key: &str,
|
src_key: &str,
|
||||||
|
src_version_id: Option<&str>,
|
||||||
range: Option<(u64, u64)>,
|
range: Option<(u64, u64)>,
|
||||||
) -> StorageResult<(String, chrono::DateTime<chrono::Utc>)>;
|
) -> StorageResult<(String, chrono::DateTime<chrono::Utc>)>;
|
||||||
|
|
||||||
@@ -148,6 +200,12 @@ pub trait StorageEngine: Send + Sync {
|
|||||||
|
|
||||||
async fn is_versioning_enabled(&self, bucket: &str) -> StorageResult<bool>;
|
async fn is_versioning_enabled(&self, bucket: &str) -> StorageResult<bool>;
|
||||||
async fn set_versioning(&self, bucket: &str, enabled: bool) -> StorageResult<()>;
|
async fn set_versioning(&self, bucket: &str, enabled: bool) -> StorageResult<()>;
|
||||||
|
async fn get_versioning_status(&self, bucket: &str) -> StorageResult<VersioningStatus>;
|
||||||
|
async fn set_versioning_status(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
status: VersioningStatus,
|
||||||
|
) -> StorageResult<()>;
|
||||||
|
|
||||||
async fn list_object_versions(
|
async fn list_object_versions(
|
||||||
&self,
|
&self,
|
||||||
@@ -166,4 +224,11 @@ pub trait StorageEngine: Send + Sync {
|
|||||||
async fn set_object_tags(&self, bucket: &str, key: &str, tags: &[Tag]) -> StorageResult<()>;
|
async fn set_object_tags(&self, bucket: &str, key: &str, tags: &[Tag]) -> StorageResult<()>;
|
||||||
|
|
||||||
async fn delete_object_tags(&self, bucket: &str, key: &str) -> StorageResult<()>;
|
async fn delete_object_tags(&self, bucket: &str, key: &str) -> StorageResult<()>;
|
||||||
|
|
||||||
|
async fn get_object_version_tags(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
) -> StorageResult<Vec<Tag>>;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -60,8 +60,15 @@ pub fn validate_object_key(
|
|||||||
return Some("Object key contains invalid segments".to_string());
|
return Some("Object key contains invalid segments".to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
if part.chars().any(|c| (c as u32) < 32) {
|
if part.len() > 255 {
|
||||||
return Some("Object key contains control characters".to_string());
|
return Some(
|
||||||
|
"Object key contains a path segment longer than 255 bytes (filesystem backend limit)"
|
||||||
|
.to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if part.contains('\0') {
|
||||||
|
return Some("Object key must not contain NUL bytes".to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
if is_windows {
|
if is_windows {
|
||||||
@@ -71,6 +78,12 @@ pub fn validate_object_key(
|
|||||||
.to_string(),
|
.to_string(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if part.chars().any(|c| (c as u32) < 32) {
|
||||||
|
return Some(
|
||||||
|
"Object key contains control characters not supported on Windows filesystems"
|
||||||
|
.to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
if part.ends_with(' ') || part.ends_with('.') {
|
if part.ends_with(' ') || part.ends_with('.') {
|
||||||
return Some(
|
return Some(
|
||||||
"Object key segments cannot end with spaces or periods on Windows".to_string(),
|
"Object key segments cannot end with spaces or periods on Windows".to_string(),
|
||||||
@@ -98,6 +111,15 @@ pub fn validate_object_key(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for part in &non_empty_parts {
|
||||||
|
if *part == ".__myfsio_dirobj__"
|
||||||
|
|| *part == ".__myfsio_empty__"
|
||||||
|
|| part.starts_with("_index.json")
|
||||||
|
{
|
||||||
|
return Some("Object key segment uses a reserved internal name".to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,10 +150,23 @@ pub fn validate_bucket_name(bucket_name: &str) -> Option<String> {
|
|||||||
return Some("Bucket name must not contain consecutive periods".to_string());
|
return Some("Bucket name must not contain consecutive periods".to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if bucket_name.contains(".-") || bucket_name.contains("-.") {
|
||||||
|
return Some(
|
||||||
|
"Bucket name must not contain a period adjacent to a hyphen".to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
if IP_REGEX.is_match(bucket_name) {
|
if IP_REGEX.is_match(bucket_name) {
|
||||||
return Some("Bucket name must not be formatted as an IP address".to_string());
|
return Some("Bucket name must not be formatted as an IP address".to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if bucket_name.starts_with("xn--") {
|
||||||
|
return Some("Bucket name must not start with the reserved prefix 'xn--'".to_string());
|
||||||
|
}
|
||||||
|
if bucket_name.ends_with("-s3alias") || bucket_name.ends_with("--ol-s3") {
|
||||||
|
return Some("Bucket name must not end with a reserved suffix".to_string());
|
||||||
|
}
|
||||||
|
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -156,6 +191,14 @@ mod tests {
|
|||||||
assert!(validate_bucket_name("192.168.1.1").is_some());
|
assert!(validate_bucket_name("192.168.1.1").is_some());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_bucket_name_period_hyphen_adjacency_rejected() {
|
||||||
|
assert!(validate_bucket_name("my-.bucket").is_some());
|
||||||
|
assert!(validate_bucket_name("my.-bucket").is_some());
|
||||||
|
assert!(validate_bucket_name("a.-b").is_some());
|
||||||
|
assert!(validate_bucket_name("a-.b").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_valid_object_keys() {
|
fn test_valid_object_keys() {
|
||||||
assert!(validate_object_key("file.txt", 1024, false, None).is_none());
|
assert!(validate_object_key("file.txt", 1024, false, None).is_none());
|
||||||
@@ -174,10 +217,18 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_object_key_max_length() {
|
fn test_object_key_max_length() {
|
||||||
let long_key = "a".repeat(1025);
|
let too_long_total = "a/".repeat(513) + "a";
|
||||||
assert!(validate_object_key(&long_key, 1024, false, None).is_some());
|
assert!(validate_object_key(&too_long_total, 1024, false, None).is_some());
|
||||||
let ok_key = "a".repeat(1024);
|
|
||||||
|
let too_long_segment = "a".repeat(256);
|
||||||
|
assert!(validate_object_key(&too_long_segment, 1024, false, None).is_some());
|
||||||
|
|
||||||
|
let ok_key = vec!["a".repeat(255); 4].join("/");
|
||||||
|
assert_eq!(ok_key.len(), 255 * 4 + 3);
|
||||||
assert!(validate_object_key(&ok_key, 1024, false, None).is_none());
|
assert!(validate_object_key(&ok_key, 1024, false, None).is_none());
|
||||||
|
|
||||||
|
let ok_max_segment = "a".repeat(255);
|
||||||
|
assert!(validate_object_key(&ok_max_segment, 1024, false, None).is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -186,4 +237,18 @@ mod tests {
|
|||||||
assert!(validate_object_key("file<name", 1024, true, None).is_some());
|
assert!(validate_object_key("file<name", 1024, true, None).is_some());
|
||||||
assert!(validate_object_key("file.txt ", 1024, true, None).is_some());
|
assert!(validate_object_key("file.txt ", 1024, true, None).is_some());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_windows_rejects_control_chars() {
|
||||||
|
assert!(validate_object_key("a\u{0001}b", 1024, true, None).is_some());
|
||||||
|
assert!(validate_object_key("a\nb", 1024, true, None).is_some());
|
||||||
|
assert!(validate_object_key("a\u{001f}b", 1024, true, None).is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_non_windows_allows_control_chars_except_nul() {
|
||||||
|
assert!(validate_object_key("a\u{0001}b", 1024, false, None).is_none());
|
||||||
|
assert!(validate_object_key("a\nb", 1024, false, None).is_none());
|
||||||
|
assert!(validate_object_key("a\0b", 1024, false, None).is_some());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,3 +8,6 @@ myfsio-common = { path = "../myfsio-common" }
|
|||||||
quick-xml = { workspace = true }
|
quick-xml = { workspace = true }
|
||||||
serde = { workspace = true }
|
serde = { workspace = true }
|
||||||
chrono = { workspace = true }
|
chrono = { workspace = true }
|
||||||
|
percent-encoding = { workspace = true }
|
||||||
|
sha2 = { workspace = true }
|
||||||
|
base64 = { workspace = true }
|
||||||
|
|||||||
@@ -1,13 +1,13 @@
|
|||||||
use quick_xml::events::Event;
|
use quick_xml::events::Event;
|
||||||
use quick_xml::Reader;
|
use quick_xml::Reader;
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default, Clone)]
|
||||||
pub struct DeleteObjectsRequest {
|
pub struct DeleteObjectsRequest {
|
||||||
pub objects: Vec<ObjectIdentifier>,
|
pub objects: Vec<ObjectIdentifier>,
|
||||||
pub quiet: bool,
|
pub quiet: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct ObjectIdentifier {
|
pub struct ObjectIdentifier {
|
||||||
pub key: String,
|
pub key: String,
|
||||||
pub version_id: Option<String>,
|
pub version_id: Option<String>,
|
||||||
@@ -86,6 +86,11 @@ pub fn parse_complete_multipart_upload(xml: &str) -> Result<CompleteMultipartUpl
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn parse_delete_objects(xml: &str) -> Result<DeleteObjectsRequest, String> {
|
pub fn parse_delete_objects(xml: &str) -> Result<DeleteObjectsRequest, String> {
|
||||||
|
let trimmed = xml.trim();
|
||||||
|
if trimmed.is_empty() {
|
||||||
|
return Err("Request body is empty".to_string());
|
||||||
|
}
|
||||||
|
|
||||||
let mut reader = Reader::from_str(xml);
|
let mut reader = Reader::from_str(xml);
|
||||||
let mut result = DeleteObjectsRequest::default();
|
let mut result = DeleteObjectsRequest::default();
|
||||||
let mut buf = Vec::new();
|
let mut buf = Vec::new();
|
||||||
@@ -93,18 +98,43 @@ pub fn parse_delete_objects(xml: &str) -> Result<DeleteObjectsRequest, String> {
|
|||||||
let mut current_key: Option<String> = None;
|
let mut current_key: Option<String> = None;
|
||||||
let mut current_version_id: Option<String> = None;
|
let mut current_version_id: Option<String> = None;
|
||||||
let mut in_object = false;
|
let mut in_object = false;
|
||||||
|
let mut saw_delete_root = false;
|
||||||
|
let mut first_element_seen = false;
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
match reader.read_event_into(&mut buf) {
|
let event = reader.read_event_into(&mut buf);
|
||||||
|
match event {
|
||||||
Ok(Event::Start(ref e)) => {
|
Ok(Event::Start(ref e)) => {
|
||||||
let name = String::from_utf8_lossy(e.name().as_ref()).to_string();
|
let name = String::from_utf8_lossy(e.name().as_ref()).to_string();
|
||||||
current_tag = name.clone();
|
current_tag = name.clone();
|
||||||
if name == "Object" {
|
if !first_element_seen {
|
||||||
|
first_element_seen = true;
|
||||||
|
if name != "Delete" {
|
||||||
|
return Err(format!(
|
||||||
|
"Expected <Delete> root element, found <{}>",
|
||||||
|
name
|
||||||
|
));
|
||||||
|
}
|
||||||
|
saw_delete_root = true;
|
||||||
|
} else if name == "Object" {
|
||||||
in_object = true;
|
in_object = true;
|
||||||
current_key = None;
|
current_key = None;
|
||||||
current_version_id = None;
|
current_version_id = None;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Ok(Event::Empty(ref e)) => {
|
||||||
|
let name = String::from_utf8_lossy(e.name().as_ref()).to_string();
|
||||||
|
if !first_element_seen {
|
||||||
|
first_element_seen = true;
|
||||||
|
if name != "Delete" {
|
||||||
|
return Err(format!(
|
||||||
|
"Expected <Delete> root element, found <{}>",
|
||||||
|
name
|
||||||
|
));
|
||||||
|
}
|
||||||
|
saw_delete_root = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
Ok(Event::Text(ref e)) => {
|
Ok(Event::Text(ref e)) => {
|
||||||
let text = e.unescape().map_err(|e| e.to_string())?.to_string();
|
let text = e.unescape().map_err(|e| e.to_string())?.to_string();
|
||||||
match current_tag.as_str() {
|
match current_tag.as_str() {
|
||||||
@@ -139,6 +169,13 @@ pub fn parse_delete_objects(xml: &str) -> Result<DeleteObjectsRequest, String> {
|
|||||||
buf.clear();
|
buf.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !saw_delete_root {
|
||||||
|
return Err("Expected <Delete> root element".to_string());
|
||||||
|
}
|
||||||
|
if result.objects.is_empty() {
|
||||||
|
return Err("Delete request must contain at least one <Object>".to_string());
|
||||||
|
}
|
||||||
|
|
||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -8,13 +8,44 @@ pub fn format_s3_datetime(dt: &DateTime<Utc>) -> String {
|
|||||||
dt.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()
|
dt.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn rate_limit_exceeded_xml() -> String {
|
pub fn rate_limit_exceeded_xml(resource: &str, request_id: &str) -> String {
|
||||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
let host_id = derive_host_id(request_id);
|
||||||
<Error><Code>SlowDown</Code><Message>Rate limit exceeded</Message></Error>"
|
format!(
|
||||||
.to_string()
|
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||||
|
<Error><Code>SlowDown</Code><Message>Please reduce your request rate</Message><Resource>{}</Resource><RequestId>{}</RequestId><HostId>{}</HostId></Error>",
|
||||||
|
xml_escape(resource),
|
||||||
|
xml_escape(request_id),
|
||||||
|
xml_escape(&host_id),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn list_buckets_xml(owner_id: &str, owner_name: &str, buckets: &[BucketMeta]) -> String {
|
fn derive_host_id(request_id: &str) -> String {
|
||||||
|
use base64::engine::general_purpose::STANDARD as B64;
|
||||||
|
use base64::Engine;
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
if request_id.is_empty() {
|
||||||
|
return String::new();
|
||||||
|
}
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
hasher.update(b"myfsio-host-id\0");
|
||||||
|
hasher.update(request_id.as_bytes());
|
||||||
|
B64.encode(hasher.finalize())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn xml_escape(s: &str) -> String {
|
||||||
|
s.replace('&', "&")
|
||||||
|
.replace('<', "<")
|
||||||
|
.replace('>', ">")
|
||||||
|
.replace('"', """)
|
||||||
|
.replace('\'', "'")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_buckets_xml(
|
||||||
|
owner_id: &str,
|
||||||
|
owner_name: &str,
|
||||||
|
buckets: &[BucketMeta],
|
||||||
|
region: &str,
|
||||||
|
) -> String {
|
||||||
let mut writer = Writer::new(Cursor::new(Vec::new()));
|
let mut writer = Writer::new(Cursor::new(Vec::new()));
|
||||||
|
|
||||||
writer
|
writer
|
||||||
@@ -47,6 +78,7 @@ pub fn list_buckets_xml(owner_id: &str, owner_name: &str, buckets: &[BucketMeta]
|
|||||||
"CreationDate",
|
"CreationDate",
|
||||||
&format_s3_datetime(&bucket.creation_date),
|
&format_s3_datetime(&bucket.creation_date),
|
||||||
);
|
);
|
||||||
|
write_text_element(&mut writer, "BucketRegion", region);
|
||||||
writer
|
writer
|
||||||
.write_event(Event::End(BytesEnd::new("Bucket")))
|
.write_event(Event::End(BytesEnd::new("Bucket")))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -62,6 +94,21 @@ pub fn list_buckets_xml(owner_id: &str, owner_name: &str, buckets: &[BucketMeta]
|
|||||||
String::from_utf8(writer.into_inner().into_inner()).unwrap()
|
String::from_utf8(writer.into_inner().into_inner()).unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn maybe_url_encode(value: &str, encoding_type: Option<&str>) -> String {
|
||||||
|
if matches!(encoding_type, Some(v) if v.eq_ignore_ascii_case("url")) {
|
||||||
|
percent_encoding::utf8_percent_encode(value, KEY_ENCODE_SET).to_string()
|
||||||
|
} else {
|
||||||
|
value.to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const KEY_ENCODE_SET: &percent_encoding::AsciiSet = &percent_encoding::NON_ALPHANUMERIC
|
||||||
|
.remove(b'-')
|
||||||
|
.remove(b'_')
|
||||||
|
.remove(b'.')
|
||||||
|
.remove(b'~')
|
||||||
|
.remove(b'/');
|
||||||
|
|
||||||
pub fn list_objects_v2_xml(
|
pub fn list_objects_v2_xml(
|
||||||
bucket_name: &str,
|
bucket_name: &str,
|
||||||
prefix: &str,
|
prefix: &str,
|
||||||
@@ -73,6 +120,36 @@ pub fn list_objects_v2_xml(
|
|||||||
continuation_token: Option<&str>,
|
continuation_token: Option<&str>,
|
||||||
next_continuation_token: Option<&str>,
|
next_continuation_token: Option<&str>,
|
||||||
key_count: usize,
|
key_count: usize,
|
||||||
|
) -> String {
|
||||||
|
list_objects_v2_xml_with_encoding(
|
||||||
|
bucket_name,
|
||||||
|
prefix,
|
||||||
|
delimiter,
|
||||||
|
max_keys,
|
||||||
|
objects,
|
||||||
|
common_prefixes,
|
||||||
|
is_truncated,
|
||||||
|
continuation_token,
|
||||||
|
next_continuation_token,
|
||||||
|
key_count,
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_objects_v2_xml_with_encoding(
|
||||||
|
bucket_name: &str,
|
||||||
|
prefix: &str,
|
||||||
|
delimiter: &str,
|
||||||
|
max_keys: usize,
|
||||||
|
objects: &[ObjectMeta],
|
||||||
|
common_prefixes: &[String],
|
||||||
|
is_truncated: bool,
|
||||||
|
continuation_token: Option<&str>,
|
||||||
|
next_continuation_token: Option<&str>,
|
||||||
|
key_count: usize,
|
||||||
|
encoding_type: Option<&str>,
|
||||||
|
fetch_owner: bool,
|
||||||
) -> String {
|
) -> String {
|
||||||
let mut writer = Writer::new(Cursor::new(Vec::new()));
|
let mut writer = Writer::new(Cursor::new(Vec::new()));
|
||||||
|
|
||||||
@@ -85,13 +162,22 @@ pub fn list_objects_v2_xml(
|
|||||||
writer.write_event(Event::Start(start)).unwrap();
|
writer.write_event(Event::Start(start)).unwrap();
|
||||||
|
|
||||||
write_text_element(&mut writer, "Name", bucket_name);
|
write_text_element(&mut writer, "Name", bucket_name);
|
||||||
write_text_element(&mut writer, "Prefix", prefix);
|
write_text_element(&mut writer, "Prefix", &maybe_url_encode(prefix, encoding_type));
|
||||||
if !delimiter.is_empty() {
|
if !delimiter.is_empty() {
|
||||||
write_text_element(&mut writer, "Delimiter", delimiter);
|
write_text_element(
|
||||||
|
&mut writer,
|
||||||
|
"Delimiter",
|
||||||
|
&maybe_url_encode(delimiter, encoding_type),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
write_text_element(&mut writer, "MaxKeys", &max_keys.to_string());
|
write_text_element(&mut writer, "MaxKeys", &max_keys.to_string());
|
||||||
write_text_element(&mut writer, "KeyCount", &key_count.to_string());
|
write_text_element(&mut writer, "KeyCount", &key_count.to_string());
|
||||||
write_text_element(&mut writer, "IsTruncated", &is_truncated.to_string());
|
write_text_element(&mut writer, "IsTruncated", &is_truncated.to_string());
|
||||||
|
if let Some(encoding) = encoding_type {
|
||||||
|
if !encoding.is_empty() {
|
||||||
|
write_text_element(&mut writer, "EncodingType", encoding);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if let Some(token) = continuation_token {
|
if let Some(token) = continuation_token {
|
||||||
write_text_element(&mut writer, "ContinuationToken", token);
|
write_text_element(&mut writer, "ContinuationToken", token);
|
||||||
@@ -104,7 +190,7 @@ pub fn list_objects_v2_xml(
|
|||||||
writer
|
writer
|
||||||
.write_event(Event::Start(BytesStart::new("Contents")))
|
.write_event(Event::Start(BytesStart::new("Contents")))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
write_text_element(&mut writer, "Key", &obj.key);
|
write_text_element(&mut writer, "Key", &maybe_url_encode(&obj.key, encoding_type));
|
||||||
write_text_element(
|
write_text_element(
|
||||||
&mut writer,
|
&mut writer,
|
||||||
"LastModified",
|
"LastModified",
|
||||||
@@ -119,6 +205,16 @@ pub fn list_objects_v2_xml(
|
|||||||
"StorageClass",
|
"StorageClass",
|
||||||
obj.storage_class.as_deref().unwrap_or("STANDARD"),
|
obj.storage_class.as_deref().unwrap_or("STANDARD"),
|
||||||
);
|
);
|
||||||
|
if fetch_owner {
|
||||||
|
writer
|
||||||
|
.write_event(Event::Start(BytesStart::new("Owner")))
|
||||||
|
.unwrap();
|
||||||
|
write_text_element(&mut writer, "ID", "myfsio");
|
||||||
|
write_text_element(&mut writer, "DisplayName", "myfsio");
|
||||||
|
writer
|
||||||
|
.write_event(Event::End(BytesEnd::new("Owner")))
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
writer
|
writer
|
||||||
.write_event(Event::End(BytesEnd::new("Contents")))
|
.write_event(Event::End(BytesEnd::new("Contents")))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -128,7 +224,7 @@ pub fn list_objects_v2_xml(
|
|||||||
writer
|
writer
|
||||||
.write_event(Event::Start(BytesStart::new("CommonPrefixes")))
|
.write_event(Event::Start(BytesStart::new("CommonPrefixes")))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
write_text_element(&mut writer, "Prefix", prefix);
|
write_text_element(&mut writer, "Prefix", &maybe_url_encode(prefix, encoding_type));
|
||||||
writer
|
writer
|
||||||
.write_event(Event::End(BytesEnd::new("CommonPrefixes")))
|
.write_event(Event::End(BytesEnd::new("CommonPrefixes")))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -151,6 +247,32 @@ pub fn list_objects_v1_xml(
|
|||||||
common_prefixes: &[String],
|
common_prefixes: &[String],
|
||||||
is_truncated: bool,
|
is_truncated: bool,
|
||||||
next_marker: Option<&str>,
|
next_marker: Option<&str>,
|
||||||
|
) -> String {
|
||||||
|
list_objects_v1_xml_with_encoding(
|
||||||
|
bucket_name,
|
||||||
|
prefix,
|
||||||
|
marker,
|
||||||
|
delimiter,
|
||||||
|
max_keys,
|
||||||
|
objects,
|
||||||
|
common_prefixes,
|
||||||
|
is_truncated,
|
||||||
|
next_marker,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_objects_v1_xml_with_encoding(
|
||||||
|
bucket_name: &str,
|
||||||
|
prefix: &str,
|
||||||
|
marker: &str,
|
||||||
|
delimiter: &str,
|
||||||
|
max_keys: usize,
|
||||||
|
objects: &[ObjectMeta],
|
||||||
|
common_prefixes: &[String],
|
||||||
|
is_truncated: bool,
|
||||||
|
next_marker: Option<&str>,
|
||||||
|
encoding_type: Option<&str>,
|
||||||
) -> String {
|
) -> String {
|
||||||
let mut writer = Writer::new(Cursor::new(Vec::new()));
|
let mut writer = Writer::new(Cursor::new(Vec::new()));
|
||||||
|
|
||||||
@@ -163,27 +285,50 @@ pub fn list_objects_v1_xml(
|
|||||||
writer.write_event(Event::Start(start)).unwrap();
|
writer.write_event(Event::Start(start)).unwrap();
|
||||||
|
|
||||||
write_text_element(&mut writer, "Name", bucket_name);
|
write_text_element(&mut writer, "Name", bucket_name);
|
||||||
write_text_element(&mut writer, "Prefix", prefix);
|
write_text_element(&mut writer, "Prefix", &maybe_url_encode(prefix, encoding_type));
|
||||||
write_text_element(&mut writer, "Marker", marker);
|
write_text_element(&mut writer, "Marker", &maybe_url_encode(marker, encoding_type));
|
||||||
write_text_element(&mut writer, "MaxKeys", &max_keys.to_string());
|
write_text_element(&mut writer, "MaxKeys", &max_keys.to_string());
|
||||||
write_text_element(&mut writer, "IsTruncated", &is_truncated.to_string());
|
write_text_element(&mut writer, "IsTruncated", &is_truncated.to_string());
|
||||||
|
|
||||||
if !delimiter.is_empty() {
|
if !delimiter.is_empty() {
|
||||||
write_text_element(&mut writer, "Delimiter", delimiter);
|
write_text_element(
|
||||||
|
&mut writer,
|
||||||
|
"Delimiter",
|
||||||
|
&maybe_url_encode(delimiter, encoding_type),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
if !delimiter.is_empty() && is_truncated {
|
if is_truncated {
|
||||||
if let Some(nm) = next_marker {
|
let fallback = next_marker
|
||||||
|
.filter(|nm| !nm.is_empty())
|
||||||
|
.map(|nm| nm.to_string())
|
||||||
|
.or_else(|| {
|
||||||
|
if !delimiter.is_empty() {
|
||||||
|
common_prefixes.last().cloned()
|
||||||
|
} else {
|
||||||
|
objects.last().map(|o| o.key.clone())
|
||||||
|
}
|
||||||
|
});
|
||||||
|
if let Some(nm) = fallback {
|
||||||
if !nm.is_empty() {
|
if !nm.is_empty() {
|
||||||
write_text_element(&mut writer, "NextMarker", nm);
|
write_text_element(
|
||||||
|
&mut writer,
|
||||||
|
"NextMarker",
|
||||||
|
&maybe_url_encode(&nm, encoding_type),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if let Some(encoding) = encoding_type {
|
||||||
|
if !encoding.is_empty() {
|
||||||
|
write_text_element(&mut writer, "EncodingType", encoding);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for obj in objects {
|
for obj in objects {
|
||||||
writer
|
writer
|
||||||
.write_event(Event::Start(BytesStart::new("Contents")))
|
.write_event(Event::Start(BytesStart::new("Contents")))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
write_text_element(&mut writer, "Key", &obj.key);
|
write_text_element(&mut writer, "Key", &maybe_url_encode(&obj.key, encoding_type));
|
||||||
write_text_element(
|
write_text_element(
|
||||||
&mut writer,
|
&mut writer,
|
||||||
"LastModified",
|
"LastModified",
|
||||||
@@ -202,7 +347,7 @@ pub fn list_objects_v1_xml(
|
|||||||
writer
|
writer
|
||||||
.write_event(Event::Start(BytesStart::new("CommonPrefixes")))
|
.write_event(Event::Start(BytesStart::new("CommonPrefixes")))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
write_text_element(&mut writer, "Prefix", cp);
|
write_text_element(&mut writer, "Prefix", &maybe_url_encode(cp, encoding_type));
|
||||||
writer
|
writer
|
||||||
.write_event(Event::End(BytesEnd::new("CommonPrefixes")))
|
.write_event(Event::End(BytesEnd::new("CommonPrefixes")))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -325,8 +470,15 @@ pub fn copy_object_result_xml(etag: &str, last_modified: &str) -> String {
|
|||||||
String::from_utf8(writer.into_inner().into_inner()).unwrap()
|
String::from_utf8(writer.into_inner().into_inner()).unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct DeletedEntry {
|
||||||
|
pub key: String,
|
||||||
|
pub version_id: Option<String>,
|
||||||
|
pub delete_marker: bool,
|
||||||
|
pub delete_marker_version_id: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
pub fn delete_result_xml(
|
pub fn delete_result_xml(
|
||||||
deleted: &[(String, Option<String>)],
|
deleted: &[DeletedEntry],
|
||||||
errors: &[(String, String, String)],
|
errors: &[(String, String, String)],
|
||||||
quiet: bool,
|
quiet: bool,
|
||||||
) -> String {
|
) -> String {
|
||||||
@@ -340,14 +492,20 @@ pub fn delete_result_xml(
|
|||||||
writer.write_event(Event::Start(start)).unwrap();
|
writer.write_event(Event::Start(start)).unwrap();
|
||||||
|
|
||||||
if !quiet {
|
if !quiet {
|
||||||
for (key, version_id) in deleted {
|
for entry in deleted {
|
||||||
writer
|
writer
|
||||||
.write_event(Event::Start(BytesStart::new("Deleted")))
|
.write_event(Event::Start(BytesStart::new("Deleted")))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
write_text_element(&mut writer, "Key", key);
|
write_text_element(&mut writer, "Key", &entry.key);
|
||||||
if let Some(vid) = version_id {
|
if let Some(ref vid) = entry.version_id {
|
||||||
write_text_element(&mut writer, "VersionId", vid);
|
write_text_element(&mut writer, "VersionId", vid);
|
||||||
}
|
}
|
||||||
|
if entry.delete_marker {
|
||||||
|
write_text_element(&mut writer, "DeleteMarker", "true");
|
||||||
|
if let Some(ref dm_vid) = entry.delete_marker_version_id {
|
||||||
|
write_text_element(&mut writer, "DeleteMarkerVersionId", dm_vid);
|
||||||
|
}
|
||||||
|
}
|
||||||
writer
|
writer
|
||||||
.write_event(Event::End(BytesEnd::new("Deleted")))
|
.write_event(Event::End(BytesEnd::new("Deleted")))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -373,10 +531,34 @@ pub fn delete_result_xml(
|
|||||||
String::from_utf8(writer.into_inner().into_inner()).unwrap()
|
String::from_utf8(writer.into_inner().into_inner()).unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct ListMultipartUploadsParams<'a> {
|
||||||
|
pub bucket: &'a str,
|
||||||
|
pub key_marker: &'a str,
|
||||||
|
pub upload_id_marker: &'a str,
|
||||||
|
pub next_key_marker: &'a str,
|
||||||
|
pub next_upload_id_marker: &'a str,
|
||||||
|
pub max_uploads: usize,
|
||||||
|
pub is_truncated: bool,
|
||||||
|
pub uploads: &'a [myfsio_common::types::MultipartUploadInfo],
|
||||||
|
}
|
||||||
|
|
||||||
pub fn list_multipart_uploads_xml(
|
pub fn list_multipart_uploads_xml(
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
uploads: &[myfsio_common::types::MultipartUploadInfo],
|
uploads: &[myfsio_common::types::MultipartUploadInfo],
|
||||||
) -> String {
|
) -> String {
|
||||||
|
list_multipart_uploads_xml_paged(&ListMultipartUploadsParams {
|
||||||
|
bucket,
|
||||||
|
key_marker: "",
|
||||||
|
upload_id_marker: "",
|
||||||
|
next_key_marker: "",
|
||||||
|
next_upload_id_marker: "",
|
||||||
|
max_uploads: 1000,
|
||||||
|
is_truncated: false,
|
||||||
|
uploads,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_multipart_uploads_xml_paged(p: &ListMultipartUploadsParams<'_>) -> String {
|
||||||
let mut writer = Writer::new(Cursor::new(Vec::new()));
|
let mut writer = Writer::new(Cursor::new(Vec::new()));
|
||||||
writer
|
writer
|
||||||
.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))
|
.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))
|
||||||
@@ -385,9 +567,17 @@ pub fn list_multipart_uploads_xml(
|
|||||||
let start = BytesStart::new("ListMultipartUploadsResult")
|
let start = BytesStart::new("ListMultipartUploadsResult")
|
||||||
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
|
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
|
||||||
writer.write_event(Event::Start(start)).unwrap();
|
writer.write_event(Event::Start(start)).unwrap();
|
||||||
write_text_element(&mut writer, "Bucket", bucket);
|
write_text_element(&mut writer, "Bucket", p.bucket);
|
||||||
|
write_text_element(&mut writer, "KeyMarker", p.key_marker);
|
||||||
|
write_text_element(&mut writer, "UploadIdMarker", p.upload_id_marker);
|
||||||
|
if p.is_truncated {
|
||||||
|
write_text_element(&mut writer, "NextKeyMarker", p.next_key_marker);
|
||||||
|
write_text_element(&mut writer, "NextUploadIdMarker", p.next_upload_id_marker);
|
||||||
|
}
|
||||||
|
write_text_element(&mut writer, "MaxUploads", &p.max_uploads.to_string());
|
||||||
|
write_text_element(&mut writer, "IsTruncated", &p.is_truncated.to_string());
|
||||||
|
|
||||||
for upload in uploads {
|
for upload in p.uploads {
|
||||||
writer
|
writer
|
||||||
.write_event(Event::Start(BytesStart::new("Upload")))
|
.write_event(Event::Start(BytesStart::new("Upload")))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -410,12 +600,36 @@ pub fn list_multipart_uploads_xml(
|
|||||||
String::from_utf8(writer.into_inner().into_inner()).unwrap()
|
String::from_utf8(writer.into_inner().into_inner()).unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct ListPartsParams<'a> {
|
||||||
|
pub bucket: &'a str,
|
||||||
|
pub key: &'a str,
|
||||||
|
pub upload_id: &'a str,
|
||||||
|
pub part_number_marker: u32,
|
||||||
|
pub next_part_number_marker: u32,
|
||||||
|
pub max_parts: usize,
|
||||||
|
pub is_truncated: bool,
|
||||||
|
pub parts: &'a [myfsio_common::types::PartMeta],
|
||||||
|
}
|
||||||
|
|
||||||
pub fn list_parts_xml(
|
pub fn list_parts_xml(
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
key: &str,
|
key: &str,
|
||||||
upload_id: &str,
|
upload_id: &str,
|
||||||
parts: &[myfsio_common::types::PartMeta],
|
parts: &[myfsio_common::types::PartMeta],
|
||||||
) -> String {
|
) -> String {
|
||||||
|
list_parts_xml_paged(&ListPartsParams {
|
||||||
|
bucket,
|
||||||
|
key,
|
||||||
|
upload_id,
|
||||||
|
part_number_marker: 0,
|
||||||
|
next_part_number_marker: parts.last().map(|p| p.part_number).unwrap_or(0),
|
||||||
|
max_parts: 1000,
|
||||||
|
is_truncated: false,
|
||||||
|
parts,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_parts_xml_paged(p: &ListPartsParams<'_>) -> String {
|
||||||
let mut writer = Writer::new(Cursor::new(Vec::new()));
|
let mut writer = Writer::new(Cursor::new(Vec::new()));
|
||||||
writer
|
writer
|
||||||
.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))
|
.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))
|
||||||
@@ -424,11 +638,23 @@ pub fn list_parts_xml(
|
|||||||
let start = BytesStart::new("ListPartsResult")
|
let start = BytesStart::new("ListPartsResult")
|
||||||
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
|
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
|
||||||
writer.write_event(Event::Start(start)).unwrap();
|
writer.write_event(Event::Start(start)).unwrap();
|
||||||
write_text_element(&mut writer, "Bucket", bucket);
|
write_text_element(&mut writer, "Bucket", p.bucket);
|
||||||
write_text_element(&mut writer, "Key", key);
|
write_text_element(&mut writer, "Key", p.key);
|
||||||
write_text_element(&mut writer, "UploadId", upload_id);
|
write_text_element(&mut writer, "UploadId", p.upload_id);
|
||||||
|
write_text_element(
|
||||||
|
&mut writer,
|
||||||
|
"PartNumberMarker",
|
||||||
|
&p.part_number_marker.to_string(),
|
||||||
|
);
|
||||||
|
write_text_element(
|
||||||
|
&mut writer,
|
||||||
|
"NextPartNumberMarker",
|
||||||
|
&p.next_part_number_marker.to_string(),
|
||||||
|
);
|
||||||
|
write_text_element(&mut writer, "MaxParts", &p.max_parts.to_string());
|
||||||
|
write_text_element(&mut writer, "IsTruncated", &p.is_truncated.to_string());
|
||||||
|
|
||||||
for part in parts {
|
for part in p.parts {
|
||||||
writer
|
writer
|
||||||
.write_event(Event::Start(BytesStart::new("Part")))
|
.write_event(Event::Start(BytesStart::new("Part")))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -461,9 +687,10 @@ mod tests {
|
|||||||
name: "test-bucket".to_string(),
|
name: "test-bucket".to_string(),
|
||||||
creation_date: Utc::now(),
|
creation_date: Utc::now(),
|
||||||
}];
|
}];
|
||||||
let xml = list_buckets_xml("owner-id", "owner-name", &buckets);
|
let xml = list_buckets_xml("owner-id", "owner-name", &buckets, "us-east-1");
|
||||||
assert!(xml.contains("<Name>test-bucket</Name>"));
|
assert!(xml.contains("<Name>test-bucket</Name>"));
|
||||||
assert!(xml.contains("<ID>owner-id</ID>"));
|
assert!(xml.contains("<ID>owner-id</ID>"));
|
||||||
|
assert!(xml.contains("<BucketRegion>us-east-1</BucketRegion>"));
|
||||||
assert!(xml.contains("ListAllMyBucketsResult"));
|
assert!(xml.contains("ListAllMyBucketsResult"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
171
docs.md
171
docs.md
@@ -122,6 +122,40 @@ These values are taken from `crates/myfsio-server/src/config.rs`.
|
|||||||
| `SECRET_KEY` | unset, then fallback to `.myfsio.sys/config/.secret` if present | Session signing and IAM config encryption key |
|
| `SECRET_KEY` | unset, then fallback to `.myfsio.sys/config/.secret` if present | Session signing and IAM config encryption key |
|
||||||
| `ADMIN_ACCESS_KEY` | unset | Optional deterministic first-run/reset access key |
|
| `ADMIN_ACCESS_KEY` | unset | Optional deterministic first-run/reset access key |
|
||||||
| `ADMIN_SECRET_KEY` | unset | Optional deterministic first-run/reset secret key |
|
| `ADMIN_SECRET_KEY` | unset | Optional deterministic first-run/reset secret key |
|
||||||
|
| `SESSION_LIFETIME_DAYS` | `1` | UI session lifetime in days |
|
||||||
|
| `LOG_LEVEL` | `INFO` | Log verbosity (also honored as `RUST_LOG`) |
|
||||||
|
| `REQUEST_BODY_TIMEOUT_SECONDS` | `60` | Per-request body read timeout |
|
||||||
|
| `MULTIPART_MIN_PART_SIZE` | `5242880` | Minimum part size enforced where applicable (5 MiB) |
|
||||||
|
| `BULK_DELETE_MAX_KEYS` | `1000` | Maximum keys per UI bulk-delete request |
|
||||||
|
| `STREAM_CHUNK_SIZE` | `1048576` | Default streaming chunk size for opt-in routes |
|
||||||
|
| `OBJECT_KEY_MAX_LENGTH_BYTES` | `1024` | Maximum object key length |
|
||||||
|
| `OBJECT_CACHE_MAX_SIZE` | `100` | Object metadata cache capacity |
|
||||||
|
| `BUCKET_CONFIG_CACHE_TTL_SECONDS` | `30` | Bucket config cache TTL |
|
||||||
|
| `OBJECT_TAG_LIMIT` | `50` | Maximum tags per object |
|
||||||
|
|
||||||
|
### Rate limiting
|
||||||
|
|
||||||
|
| Variable | Default | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `RATE_LIMIT_DEFAULT` | `5000 per minute` | Default S3 / KMS rate limit. Accepts `N per <s/m/h/d>` or `N/<seconds>` |
|
||||||
|
| `RATE_LIMIT_LIST_BUCKETS` | inherits `RATE_LIMIT_DEFAULT` | Override for `GET /` |
|
||||||
|
| `RATE_LIMIT_BUCKET_OPS` | inherits `RATE_LIMIT_DEFAULT` | Override for `/{bucket}` |
|
||||||
|
| `RATE_LIMIT_OBJECT_OPS` | inherits `RATE_LIMIT_DEFAULT` | Override for `/{bucket}/{key}` |
|
||||||
|
| `RATE_LIMIT_HEAD_OPS` | inherits `RATE_LIMIT_DEFAULT` | Override for HEAD requests |
|
||||||
|
| `RATE_LIMIT_ADMIN` | `60 per minute` | Override for `/admin/*` |
|
||||||
|
| `RATE_LIMIT_STORAGE_URI` | `memory://` | Backend for rate-limit state. Only `memory://` is supported today |
|
||||||
|
|
||||||
|
### CORS and proxying
|
||||||
|
|
||||||
|
| Variable | Default | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `CORS_ORIGINS` | `*` | Server-level allowed origins (comma-separated) |
|
||||||
|
| `CORS_METHODS` | `GET,PUT,POST,DELETE,OPTIONS,HEAD` | Server-level allowed methods |
|
||||||
|
| `CORS_ALLOW_HEADERS` | `*` | Allowed request headers |
|
||||||
|
| `CORS_EXPOSE_HEADERS` | `*` | Headers exposed to the browser |
|
||||||
|
| `NUM_TRUSTED_PROXIES` | `0` | Trusted reverse-proxy count. Forwarded-IP headers are ignored when `0` |
|
||||||
|
| `ALLOWED_REDIRECT_HOSTS` | empty | Comma-separated whitelist of safe UI login redirect hosts |
|
||||||
|
| `ALLOW_INTERNAL_ENDPOINTS` | `false` | Gate for internal diagnostic routes |
|
||||||
|
|
||||||
### Feature toggles
|
### Feature toggles
|
||||||
|
|
||||||
@@ -131,6 +165,12 @@ These values are taken from `crates/myfsio-server/src/config.rs`.
|
|||||||
| `KMS_ENABLED` | `false` | Enable built-in KMS support |
|
| `KMS_ENABLED` | `false` | Enable built-in KMS support |
|
||||||
| `GC_ENABLED` | `false` | Start the garbage collector worker |
|
| `GC_ENABLED` | `false` | Start the garbage collector worker |
|
||||||
| `INTEGRITY_ENABLED` | `false` | Start the integrity worker |
|
| `INTEGRITY_ENABLED` | `false` | Start the integrity worker |
|
||||||
|
| `INTEGRITY_AUTO_HEAL` | `false` | When the periodic scan finishes, attempt to heal each issue (peer-fetch corrupted bytes, drop phantom metadata, etc.) |
|
||||||
|
| `INTEGRITY_DRY_RUN` | `false` | Report what the periodic scan would heal without touching anything |
|
||||||
|
| `INTEGRITY_INTERVAL_HOURS` | `24` | Period between background integrity scans |
|
||||||
|
| `INTEGRITY_BATCH_SIZE` | `10000` | Max objects scanned per cycle |
|
||||||
|
| `INTEGRITY_HEAL_CONCURRENCY` | `4` | Max concurrent heal tasks per cycle |
|
||||||
|
| `INTEGRITY_QUARANTINE_RETENTION_DAYS` | `7` | How long to retain quarantined files (cleaned up by GC) |
|
||||||
| `LIFECYCLE_ENABLED` | `false` | Start the lifecycle worker |
|
| `LIFECYCLE_ENABLED` | `false` | Start the lifecycle worker |
|
||||||
| `METRICS_HISTORY_ENABLED` | `false` | Persist system metrics snapshots |
|
| `METRICS_HISTORY_ENABLED` | `false` | Persist system metrics snapshots |
|
||||||
| `OPERATION_METRICS_ENABLED` | `false` | Persist API operation metrics |
|
| `OPERATION_METRICS_ENABLED` | `false` | Persist API operation metrics |
|
||||||
@@ -162,15 +202,55 @@ These values are taken from `crates/myfsio-server/src/config.rs`.
|
|||||||
| `SITE_SYNC_MAX_RETRIES` | `2` | Site sync retry count |
|
| `SITE_SYNC_MAX_RETRIES` | `2` | Site sync retry count |
|
||||||
| `SITE_SYNC_CLOCK_SKEW_TOLERANCE_SECONDS` | `1.0` | Allowed skew between peers |
|
| `SITE_SYNC_CLOCK_SKEW_TOLERANCE_SECONDS` | `1.0` | Allowed skew between peers |
|
||||||
|
|
||||||
|
### Garbage collection
|
||||||
|
|
||||||
|
| Variable | Default | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `GC_INTERVAL_HOURS` | `6` | Hours between GC cycles |
|
||||||
|
| `GC_TEMP_FILE_MAX_AGE_HOURS` | `24` | Delete temp files older than this |
|
||||||
|
| `GC_MULTIPART_MAX_AGE_DAYS` | `7` | Delete orphaned multipart uploads older than this |
|
||||||
|
| `GC_LOCK_FILE_MAX_AGE_HOURS` | `1` | Delete stale lock files older than this |
|
||||||
|
| `GC_DRY_RUN` | `false` | Log deletions without removing files |
|
||||||
|
|
||||||
|
### Encryption tuning
|
||||||
|
|
||||||
|
| Variable | Default | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `ENCRYPTION_CHUNK_SIZE_BYTES` | `65536` | Plaintext chunk size for streaming AES-256-GCM (64 KiB) |
|
||||||
|
| `KMS_GENERATE_DATA_KEY_MIN_BYTES` | `1` | Minimum size for `generate-data-key` |
|
||||||
|
| `KMS_GENERATE_DATA_KEY_MAX_BYTES` | `1024` | Maximum size for `generate-data-key` |
|
||||||
|
| `LIFECYCLE_MAX_HISTORY_PER_BUCKET` | `50` | Max lifecycle history records kept per bucket |
|
||||||
|
|
||||||
### Site identity values used by the UI
|
### Site identity values used by the UI
|
||||||
|
|
||||||
These are read directly by UI pages:
|
These are read directly by UI pages:
|
||||||
|
|
||||||
| Variable | Description |
|
| Variable | Default | Description |
|
||||||
| --- | --- |
|
| --- | --- | --- |
|
||||||
| `SITE_ID` | Local site identifier shown in the UI |
|
| `SITE_ID` | unset | Local site identifier shown in the UI |
|
||||||
| `SITE_ENDPOINT` | Public endpoint for this site |
|
| `SITE_ENDPOINT` | unset | Public endpoint for this site |
|
||||||
| `SITE_REGION` | Display region for the local site |
|
| `SITE_REGION` | matches `AWS_REGION` | Display region for the local site |
|
||||||
|
| `SITE_PRIORITY` | `100` | Routing priority (lower = preferred) |
|
||||||
|
|
||||||
|
### Cross-site authentication
|
||||||
|
|
||||||
|
The Cluster page on each site fetches `/admin/cluster/overview` from every registered peer to render their cards. That endpoint is gated by `require_admin_or_registered_peer`, which accepts a request when **either**:
|
||||||
|
|
||||||
|
1. The signing access key is a full admin on the receiving site (policy `{"bucket":"*","actions":["*"]}`), **or**
|
||||||
|
2. The signing access key matches the **Peer Inbound Access Key** field on the receiving site's site-registry entry for that peer.
|
||||||
|
|
||||||
|
Option 2 is the least-privilege path and is what the Sites UI exposes per peer. The value goes into the *receiving* peer entry and is **the access key the other site signs with when calling here** — i.e. copied from the *other* site's outbound Connection that targets this site.
|
||||||
|
|
||||||
|
Symmetric setup for two sites `us-east-1` and `us-west-1`:
|
||||||
|
|
||||||
|
| On site | Peer entry | Peer Inbound Access Key |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `us-east-1` | `us-west-1` | the access key in **us-west-1's** outbound Connection that points at us-east-1 |
|
||||||
|
| `us-west-1` | `us-east-1` | the access key in **us-east-1's** outbound Connection that points at us-west-1 |
|
||||||
|
|
||||||
|
Putting your own admin key into your own peer entry does nothing — the inbound caller is the peer, not you. The whitelisted access key must still exist as an enabled IAM user on the receiving site so SigV4 verification finds a matching secret. Its policy can be empty for cluster-overview alone; for site-sync it needs the S3 verbs the sync workload uses (`list`, `read`, plus `write`/`delete` for replicated/bidirectional buckets). Leave the field blank only if the signing key is a real admin on the receiving site.
|
||||||
|
|
||||||
|
The in-app **Documentation → Site Registry** page has a worked example with side-by-side cards.
|
||||||
|
|
||||||
## 7. Data Layout
|
## 7. Data Layout
|
||||||
|
|
||||||
@@ -178,29 +258,39 @@ With the default `STORAGE_ROOT=./data`, the Rust server writes:
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
data/
|
data/
|
||||||
<bucket>/
|
<bucket>/ # raw object data
|
||||||
.myfsio.sys/
|
.myfsio.sys/
|
||||||
config/
|
config/
|
||||||
iam.json
|
.secret # persisted SECRET_KEY (if generated)
|
||||||
bucket_policies.json
|
iam.json # IAM users / access keys / policies
|
||||||
connections.json
|
bucket_policies.json # legacy bucket policies (fallback only)
|
||||||
gc_history.json
|
connections.json # remote endpoint credentials
|
||||||
integrity_history.json
|
replication_rules.json # replication rules
|
||||||
metrics_history.json
|
site_registry.json # local site + peer registry
|
||||||
operation_metrics.json
|
website_domains.json # domain → bucket mapping (if enabled)
|
||||||
|
gc_history.json # GC execution history (if enabled)
|
||||||
|
integrity_history.json # integrity scan history (if enabled)
|
||||||
|
metrics_history.json # system metrics history (if enabled)
|
||||||
|
operation_metrics.json # API operation metrics (if enabled)
|
||||||
buckets/<bucket>/
|
buckets/<bucket>/
|
||||||
meta/
|
.bucket.json # bucket config (versioning, cors, lifecycle, etc.)
|
||||||
versions/
|
meta/ # per-object metadata
|
||||||
multipart/
|
versions/ # archived versions (if versioning enabled)
|
||||||
|
lifecycle_history.json # lifecycle action log (if any rule has fired)
|
||||||
|
replication_failures.json # bounded failure log
|
||||||
|
site_sync_state.json # bidi sync watermark
|
||||||
|
multipart/ # in-progress multipart uploads
|
||||||
keys/
|
keys/
|
||||||
|
kms_master.key # 32-byte master key (base64)
|
||||||
|
kms_keys.json # KMS keys, encrypted under master key
|
||||||
```
|
```
|
||||||
|
|
||||||
Important files:
|
Notable files:
|
||||||
|
|
||||||
- `data/.myfsio.sys/config/iam.json`: IAM users, access keys, and inline policies
|
- `iam.json` is Fernet-encrypted at rest when `SECRET_KEY` is set.
|
||||||
- `data/.myfsio.sys/config/bucket_policies.json`: bucket policies
|
- `bucket_policies.json` is read only as a fallback for policies that pre-date per-bucket `.bucket.json`.
|
||||||
- `data/.myfsio.sys/config/connections.json`: replication connection settings
|
- `kms_master.key` is plaintext on disk — protect `keys/` with filesystem permissions.
|
||||||
- `data/.myfsio.sys/config/.secret`: persisted secret key when one has been generated for the install
|
- `*_history.json` files only appear when their owning service has been enabled at least once.
|
||||||
|
|
||||||
## 8. Background Services
|
## 8. Background Services
|
||||||
|
|
||||||
@@ -230,14 +320,17 @@ Enable with:
|
|||||||
GC_ENABLED=true cargo run -p myfsio-server --
|
GC_ENABLED=true cargo run -p myfsio-server --
|
||||||
```
|
```
|
||||||
|
|
||||||
Current Rust defaults from `GcConfig::default()`:
|
Defaults (override with the env vars in section 6):
|
||||||
|
|
||||||
- Run every 6 hours
|
- `GC_INTERVAL_HOURS=6`
|
||||||
- Temp files older than 24 hours are eligible for cleanup
|
- `GC_TEMP_FILE_MAX_AGE_HOURS=24`
|
||||||
- Multipart uploads older than 7 days are eligible for cleanup
|
- `GC_MULTIPART_MAX_AGE_DAYS=7`
|
||||||
- Lock files older than 1 hour are eligible for cleanup
|
- `GC_LOCK_FILE_MAX_AGE_HOURS=1`
|
||||||
|
- `GC_DRY_RUN=false`
|
||||||
|
|
||||||
Those GC timings are currently hardcoded defaults, not environment-driven configuration.
|
Each GC cycle also sweeps `data/.myfsio.sys/quarantine/<bucket>/<ts>/` directories whose `<ts>` mtime is older than `INTEGRITY_QUARANTINE_RETENTION_DAYS`, freeing the bytes recorded in `quarantine_bytes_freed` / `quarantine_entries_deleted` in the result JSON.
|
||||||
|
|
||||||
|
History is persisted at `data/.myfsio.sys/config/gc_history.json` and can be triggered manually via `POST /admin/gc/run` (use `{"dry_run": true}` to preview).
|
||||||
|
|
||||||
### Integrity scanning
|
### Integrity scanning
|
||||||
|
|
||||||
@@ -247,11 +340,27 @@ Enable with:
|
|||||||
INTEGRITY_ENABLED=true cargo run -p myfsio-server --
|
INTEGRITY_ENABLED=true cargo run -p myfsio-server --
|
||||||
```
|
```
|
||||||
|
|
||||||
Current Rust defaults from `IntegrityConfig::default()`:
|
Tune with:
|
||||||
|
|
||||||
- Run every 24 hours
|
```bash
|
||||||
- Batch size 1000
|
INTEGRITY_INTERVAL_HOURS=24
|
||||||
- Auto-heal disabled
|
INTEGRITY_BATCH_SIZE=10000
|
||||||
|
INTEGRITY_AUTO_HEAL=false
|
||||||
|
INTEGRITY_DRY_RUN=false
|
||||||
|
INTEGRITY_HEAL_CONCURRENCY=4
|
||||||
|
INTEGRITY_QUARANTINE_RETENTION_DAYS=7
|
||||||
|
```
|
||||||
|
|
||||||
|
When `INTEGRITY_AUTO_HEAL=true` (and `INTEGRITY_DRY_RUN=false`), each scan ends with a heal phase that processes the issues it just recorded. For `corrupted_object` the bad bytes are renamed into `data/.myfsio.sys/quarantine/<bucket>/<ts>/<key>` and the heal logic tries, in order:
|
||||||
|
|
||||||
|
1. **Pull from peer.** If a replication rule for the bucket points at a healthy remote whose `HEAD` returns the same ETag the local index has, the body is streamed to a temp file, MD5-verified against the stored ETag, and atomically swapped into the live path. The poison flags are cleared on success.
|
||||||
|
2. **Poison the entry.** If there is no replication target, the peer disagrees on the ETag, the peer is unreachable, or the downloaded body fails verification, the index entry is mutated to add `__corrupted__: "true"`, `__corrupted_at__`, `__corruption_detail__`, and `__quarantine_path__`. The data file stays in quarantine for `INTEGRITY_QUARANTINE_RETENTION_DAYS`.
|
||||||
|
|
||||||
|
Subsequent reads (`GET`, `HEAD`, `CopyObject` source) on a poisoned key return `422 ObjectCorrupted` instead of serving rotted bytes; the response includes an `x-amz-error-code: ObjectCorrupted` header so HEAD callers (which receive no body) can still detect the condition. Replication push skips poisoned keys; subsequent integrity scans skip poisoned keys instead of re-flagging them. Overwriting the key with a fresh `PUT` clears the poison.
|
||||||
|
|
||||||
|
`stale_version`, `etag_cache_inconsistency`, and `phantom_metadata` issues are healed locally (move-to-quarantine, rebuild cache, drop entry); `orphaned_object` is reported only.
|
||||||
|
|
||||||
|
Override per-invocation by passing `auto_heal` / `dry_run` to `POST /admin/integrity/run`. The response and history records now include a `heal_stats` map keyed by issue type with `{found, healed, poisoned, peer_mismatch, peer_unavailable, verify_failed, failed, skipped}`. History is at `data/.myfsio.sys/config/integrity_history.json`.
|
||||||
|
|
||||||
### Metrics history
|
### Metrics history
|
||||||
|
|
||||||
|
|||||||
@@ -11,11 +11,12 @@
|
|||||||
# --data-dir DIR Data directory (default: /var/lib/myfsio)
|
# --data-dir DIR Data directory (default: /var/lib/myfsio)
|
||||||
# --log-dir DIR Log directory (default: /var/log/myfsio)
|
# --log-dir DIR Log directory (default: /var/log/myfsio)
|
||||||
# --user USER System user to run as (default: myfsio)
|
# --user USER System user to run as (default: myfsio)
|
||||||
|
# --host HOST Bind host (default: 0.0.0.0)
|
||||||
# --port PORT API port (default: 5000)
|
# --port PORT API port (default: 5000)
|
||||||
# --ui-port PORT UI port (default: 5100)
|
# --ui-port PORT UI port (default: 5100)
|
||||||
# --api-url URL Public API URL (for presigned URLs behind proxy)
|
# --api-url URL Public API URL (for presigned URLs behind proxy)
|
||||||
# --no-systemd Skip systemd service creation
|
# --no-systemd Skip systemd service creation
|
||||||
# --binary PATH Path to myfsio binary (will download if not provided)
|
# --binary PATH Path to myfsio binary (default: ./myfsio)
|
||||||
# -y, --yes Skip confirmation prompts
|
# -y, --yes Skip confirmation prompts
|
||||||
#
|
#
|
||||||
|
|
||||||
@@ -25,6 +26,7 @@ INSTALL_DIR="/opt/myfsio"
|
|||||||
DATA_DIR="/var/lib/myfsio"
|
DATA_DIR="/var/lib/myfsio"
|
||||||
LOG_DIR="/var/log/myfsio"
|
LOG_DIR="/var/log/myfsio"
|
||||||
SERVICE_USER="myfsio"
|
SERVICE_USER="myfsio"
|
||||||
|
BIND_HOST="0.0.0.0"
|
||||||
API_PORT="5000"
|
API_PORT="5000"
|
||||||
UI_PORT="5100"
|
UI_PORT="5100"
|
||||||
API_URL=""
|
API_URL=""
|
||||||
@@ -34,54 +36,19 @@ AUTO_YES=false
|
|||||||
|
|
||||||
while [[ $# -gt 0 ]]; do
|
while [[ $# -gt 0 ]]; do
|
||||||
case $1 in
|
case $1 in
|
||||||
--install-dir)
|
--install-dir) INSTALL_DIR="$2"; shift 2 ;;
|
||||||
INSTALL_DIR="$2"
|
--data-dir) DATA_DIR="$2"; shift 2 ;;
|
||||||
shift 2
|
--log-dir) LOG_DIR="$2"; shift 2 ;;
|
||||||
;;
|
--user) SERVICE_USER="$2"; shift 2 ;;
|
||||||
--data-dir)
|
--host) BIND_HOST="$2"; shift 2 ;;
|
||||||
DATA_DIR="$2"
|
--port) API_PORT="$2"; shift 2 ;;
|
||||||
shift 2
|
--ui-port) UI_PORT="$2"; shift 2 ;;
|
||||||
;;
|
--api-url) API_URL="$2"; shift 2 ;;
|
||||||
--log-dir)
|
--no-systemd) SKIP_SYSTEMD=true; shift ;;
|
||||||
LOG_DIR="$2"
|
--binary) BINARY_PATH="$2"; shift 2 ;;
|
||||||
shift 2
|
-y|--yes) AUTO_YES=true; shift ;;
|
||||||
;;
|
-h|--help) head -22 "$0" | tail -17; exit 0 ;;
|
||||||
--user)
|
*) echo "Unknown option: $1"; exit 1 ;;
|
||||||
SERVICE_USER="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--port)
|
|
||||||
API_PORT="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--ui-port)
|
|
||||||
UI_PORT="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--api-url)
|
|
||||||
API_URL="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--no-systemd)
|
|
||||||
SKIP_SYSTEMD=true
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
--binary)
|
|
||||||
BINARY_PATH="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
-y|--yes)
|
|
||||||
AUTO_YES=true
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
-h|--help)
|
|
||||||
head -30 "$0" | tail -25
|
|
||||||
exit 0
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "Unknown option: $1"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
@@ -107,14 +74,11 @@ echo " Install directory: $INSTALL_DIR"
|
|||||||
echo " Data directory: $DATA_DIR"
|
echo " Data directory: $DATA_DIR"
|
||||||
echo " Log directory: $LOG_DIR"
|
echo " Log directory: $LOG_DIR"
|
||||||
echo " Service user: $SERVICE_USER"
|
echo " Service user: $SERVICE_USER"
|
||||||
|
echo " Bind host: $BIND_HOST"
|
||||||
echo " API port: $API_PORT"
|
echo " API port: $API_PORT"
|
||||||
echo " UI port: $UI_PORT"
|
echo " UI port: $UI_PORT"
|
||||||
if [[ -n "$API_URL" ]]; then
|
[[ -n "$API_URL" ]] && echo " Public API URL: $API_URL"
|
||||||
echo " Public API URL: $API_URL"
|
[[ -n "$BINARY_PATH" ]] && echo " Binary: $BINARY_PATH"
|
||||||
fi
|
|
||||||
if [[ -n "$BINARY_PATH" ]]; then
|
|
||||||
echo " Binary path: $BINARY_PATH"
|
|
||||||
fi
|
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [[ "$AUTO_YES" != true ]]; then
|
if [[ "$AUTO_YES" != true ]]; then
|
||||||
@@ -143,12 +107,9 @@ echo "------------------------------------------------------------"
|
|||||||
echo "STEP 3: Creating Directories"
|
echo "STEP 3: Creating Directories"
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
echo ""
|
echo ""
|
||||||
mkdir -p "$INSTALL_DIR"
|
mkdir -p "$INSTALL_DIR" && echo " [OK] Created $INSTALL_DIR"
|
||||||
echo " [OK] Created $INSTALL_DIR"
|
mkdir -p "$DATA_DIR" && echo " [OK] Created $DATA_DIR"
|
||||||
mkdir -p "$DATA_DIR"
|
mkdir -p "$LOG_DIR" && echo " [OK] Created $LOG_DIR"
|
||||||
echo " [OK] Created $DATA_DIR"
|
|
||||||
mkdir -p "$LOG_DIR"
|
|
||||||
echo " [OK] Created $LOG_DIR"
|
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
@@ -156,13 +117,12 @@ echo "STEP 4: Installing Binary"
|
|||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
echo ""
|
echo ""
|
||||||
if [[ -n "$BINARY_PATH" ]]; then
|
if [[ -n "$BINARY_PATH" ]]; then
|
||||||
if [[ -f "$BINARY_PATH" ]]; then
|
if [[ ! -f "$BINARY_PATH" ]]; then
|
||||||
cp "$BINARY_PATH" "$INSTALL_DIR/myfsio"
|
|
||||||
echo " [OK] Copied binary from $BINARY_PATH"
|
|
||||||
else
|
|
||||||
echo " [ERROR] Binary not found at $BINARY_PATH"
|
echo " [ERROR] Binary not found at $BINARY_PATH"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
cp "$BINARY_PATH" "$INSTALL_DIR/myfsio"
|
||||||
|
echo " [OK] Copied binary from $BINARY_PATH"
|
||||||
elif [[ -f "./myfsio" ]]; then
|
elif [[ -f "./myfsio" ]]; then
|
||||||
cp "./myfsio" "$INSTALL_DIR/myfsio"
|
cp "./myfsio" "$INSTALL_DIR/myfsio"
|
||||||
echo " [OK] Copied binary from ./myfsio"
|
echo " [OK] Copied binary from ./myfsio"
|
||||||
@@ -173,20 +133,53 @@ else
|
|||||||
fi
|
fi
|
||||||
chmod +x "$INSTALL_DIR/myfsio"
|
chmod +x "$INSTALL_DIR/myfsio"
|
||||||
echo " [OK] Set executable permissions"
|
echo " [OK] Set executable permissions"
|
||||||
|
echo " [INFO] Templates and static UI assets are embedded in the binary"
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
echo "STEP 5: Generating Secret Key"
|
echo "STEP 5: Creating Configuration File"
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
echo ""
|
echo ""
|
||||||
SECRET_KEY=$(openssl rand -base64 32)
|
|
||||||
echo " [OK] Generated secure SECRET_KEY"
|
|
||||||
|
|
||||||
echo ""
|
SECRET_FILE="$DATA_DIR/.myfsio.sys/config/.secret"
|
||||||
echo "------------------------------------------------------------"
|
mkdir -p "$(dirname "$SECRET_FILE")"
|
||||||
echo "STEP 6: Creating Configuration File"
|
if [[ -s "$SECRET_FILE" ]]; then
|
||||||
echo "------------------------------------------------------------"
|
echo " [OK] Existing secret found at $SECRET_FILE - preserving"
|
||||||
echo ""
|
elif [[ -n "${SECRET_KEY:-}" ]]; then
|
||||||
|
printf '%s' "$SECRET_KEY" > "$SECRET_FILE"
|
||||||
|
chmod 600 "$SECRET_FILE"
|
||||||
|
echo " [OK] Wrote SECRET_KEY from environment to $SECRET_FILE"
|
||||||
|
else
|
||||||
|
if command -v openssl &>/dev/null; then
|
||||||
|
printf '%s' "$(openssl rand -base64 32)" > "$SECRET_FILE"
|
||||||
|
elif [[ -r /dev/urandom ]]; then
|
||||||
|
printf '%s' "$(head -c 32 /dev/urandom | base64)" > "$SECRET_FILE"
|
||||||
|
else
|
||||||
|
echo " [ERROR] Neither openssl nor /dev/urandom available; cannot generate secret"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
chmod 600 "$SECRET_FILE"
|
||||||
|
echo " [OK] Generated secret key at $SECRET_FILE"
|
||||||
|
fi
|
||||||
|
unset SECRET_KEY
|
||||||
|
|
||||||
|
if [[ -n "$API_URL" ]]; then
|
||||||
|
EFFECTIVE_API_URL="$API_URL"
|
||||||
|
else
|
||||||
|
case "$BIND_HOST" in
|
||||||
|
0.0.0.0|::|"")
|
||||||
|
DETECTED_IP=$(hostname -I 2>/dev/null | awk '{print $1}')
|
||||||
|
[[ -z "$DETECTED_IP" ]] && DETECTED_IP="127.0.0.1"
|
||||||
|
EFFECTIVE_API_URL="http://$DETECTED_IP:$API_PORT"
|
||||||
|
echo " [INFO] Bind host is $BIND_HOST; deriving API_BASE_URL=$EFFECTIVE_API_URL"
|
||||||
|
echo " Pass --api-url to set a public URL for presigned access."
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
EFFECTIVE_API_URL="http://$BIND_HOST:$API_PORT"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|
||||||
cat > "$INSTALL_DIR/myfsio.env" << EOF
|
cat > "$INSTALL_DIR/myfsio.env" << EOF
|
||||||
# MyFSIO Configuration
|
# MyFSIO Configuration
|
||||||
# Generated by install.sh on $(date)
|
# Generated by install.sh on $(date)
|
||||||
@@ -196,61 +189,63 @@ cat > "$INSTALL_DIR/myfsio.env" << EOF
|
|||||||
# STORAGE PATHS
|
# STORAGE PATHS
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
STORAGE_ROOT=$DATA_DIR
|
STORAGE_ROOT=$DATA_DIR
|
||||||
LOG_DIR=$LOG_DIR
|
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# NETWORK
|
# NETWORK
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
APP_HOST=0.0.0.0
|
HOST=$BIND_HOST
|
||||||
APP_PORT=$API_PORT
|
PORT=$API_PORT
|
||||||
|
UI_PORT=$UI_PORT
|
||||||
|
|
||||||
# Public URL (set this if behind a reverse proxy for presigned URLs)
|
# Public URL used to sign presigned URLs (override with --api-url for proxies)
|
||||||
$(if [[ -n "$API_URL" ]]; then echo "API_BASE_URL=$API_URL"; else echo "# API_BASE_URL=https://s3.example.com"; fi)
|
API_BASE_URL=$EFFECTIVE_API_URL
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# SECURITY
|
# SECURITY
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# Secret key for session signing (auto-generated if not set)
|
|
||||||
SECRET_KEY=$SECRET_KEY
|
|
||||||
|
|
||||||
# CORS settings - restrict in production
|
# CORS settings - restrict in production
|
||||||
CORS_ORIGINS=*
|
CORS_ORIGINS=*
|
||||||
|
# CORS_METHODS=GET,PUT,POST,DELETE,OPTIONS,HEAD
|
||||||
|
# CORS_ALLOW_HEADERS=*
|
||||||
|
# CORS_EXPOSE_HEADERS=*
|
||||||
|
|
||||||
# Brute-force protection
|
# Reverse proxy settings (number of trusted proxies in front)
|
||||||
AUTH_MAX_ATTEMPTS=5
|
|
||||||
AUTH_LOCKOUT_MINUTES=15
|
|
||||||
|
|
||||||
# Reverse proxy settings (set to number of trusted proxies in front)
|
|
||||||
# NUM_TRUSTED_PROXIES=1
|
# NUM_TRUSTED_PROXIES=1
|
||||||
|
|
||||||
# Allow internal admin endpoints (only enable on trusted networks)
|
# Allow internal/diagnostic admin endpoints (only on trusted networks)
|
||||||
# ALLOW_INTERNAL_ENDPOINTS=false
|
# ALLOW_INTERNAL_ENDPOINTS=false
|
||||||
|
|
||||||
# Allowed hosts for redirects (comma-separated, empty = restrict all)
|
# Comma-separated external hosts allowed for UI login redirects
|
||||||
# ALLOWED_REDIRECT_HOSTS=
|
# ALLOWED_REDIRECT_HOSTS=
|
||||||
|
|
||||||
|
# UI session lifetime in days
|
||||||
|
# SESSION_LIFETIME_DAYS=1
|
||||||
|
|
||||||
|
# SigV4 timestamp tolerance (seconds)
|
||||||
|
# SIGV4_TIMESTAMP_TOLERANCE_SECONDS=900
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# UI ASSET OVERRIDES (optional - assets are embedded in the binary by default)
|
||||||
|
# =============================================================================
|
||||||
|
# Set these only when developing UI changes against an unpacked source tree.
|
||||||
|
# TEMPLATES_DIR=/path/to/templates
|
||||||
|
# STATIC_DIR=/path/to/static
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# LOGGING
|
# LOGGING
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
LOG_LEVEL=INFO
|
LOG_LEVEL=INFO
|
||||||
LOG_TO_FILE=true
|
# RUST_LOG=info,myfsio_server=info
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# RATE LIMITING
|
# RATE LIMITING
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
RATE_LIMIT_DEFAULT=200 per minute
|
RATE_LIMIT_DEFAULT=500 per minute
|
||||||
# RATE_LIMIT_LIST_BUCKETS=60 per minute
|
# RATE_LIMIT_LIST_BUCKETS=500 per minute
|
||||||
# RATE_LIMIT_BUCKET_OPS=120 per minute
|
# RATE_LIMIT_BUCKET_OPS=500 per minute
|
||||||
# RATE_LIMIT_OBJECT_OPS=240 per minute
|
# RATE_LIMIT_OBJECT_OPS=500 per minute
|
||||||
# RATE_LIMIT_ADMIN=60 per minute
|
# RATE_LIMIT_HEAD_OPS=500 per minute
|
||||||
|
RATE_LIMIT_ADMIN=60 per minute
|
||||||
# =============================================================================
|
|
||||||
# SERVER TUNING (0 = auto-detect based on system resources)
|
|
||||||
# =============================================================================
|
|
||||||
# SERVER_THREADS=0
|
|
||||||
# SERVER_CONNECTION_LIMIT=0
|
|
||||||
# SERVER_BACKLOG=0
|
|
||||||
# SERVER_CHANNEL_TIMEOUT=120
|
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# ENCRYPTION (uncomment to enable)
|
# ENCRYPTION (uncomment to enable)
|
||||||
@@ -259,39 +254,50 @@ RATE_LIMIT_DEFAULT=200 per minute
|
|||||||
# KMS_ENABLED=true
|
# KMS_ENABLED=true
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# SITE SYNC / REPLICATION (for multi-site deployments)
|
# SITE SYNC / REPLICATION (multi-site deployments)
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# SITE_ID=site-1
|
# SITE_ID=site-1
|
||||||
# SITE_ENDPOINT=https://s3-site1.example.com
|
# SITE_ENDPOINT=https://s3-site1.example.com
|
||||||
# SITE_REGION=us-east-1
|
# SITE_REGION=us-east-1
|
||||||
|
# SITE_PRIORITY=100
|
||||||
# SITE_SYNC_ENABLED=false
|
# SITE_SYNC_ENABLED=false
|
||||||
|
# SITE_SYNC_INTERVAL_SECONDS=60
|
||||||
|
# SITE_SYNC_BATCH_SIZE=100
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# OPTIONAL FEATURES
|
# OPTIONAL FEATURES
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
# WEBSITE_HOSTING_ENABLED=false
|
||||||
# LIFECYCLE_ENABLED=false
|
# LIFECYCLE_ENABLED=false
|
||||||
# METRICS_HISTORY_ENABLED=false
|
# METRICS_HISTORY_ENABLED=false
|
||||||
# OPERATION_METRICS_ENABLED=false
|
# OPERATION_METRICS_ENABLED=false
|
||||||
|
# GC_ENABLED=false
|
||||||
|
# GC_INTERVAL_HOURS=6
|
||||||
|
# GC_DRY_RUN=false
|
||||||
|
# INTEGRITY_ENABLED=false
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# FIRST-RUN ADMIN OVERRIDE (optional)
|
||||||
|
# =============================================================================
|
||||||
|
# ADMIN_ACCESS_KEY=
|
||||||
|
# ADMIN_SECRET_KEY=
|
||||||
EOF
|
EOF
|
||||||
chmod 600 "$INSTALL_DIR/myfsio.env"
|
chmod 600 "$INSTALL_DIR/myfsio.env"
|
||||||
echo " [OK] Created $INSTALL_DIR/myfsio.env"
|
echo " [OK] Created $INSTALL_DIR/myfsio.env"
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
echo "STEP 7: Setting Permissions"
|
echo "STEP 6: Setting Permissions"
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
echo ""
|
echo ""
|
||||||
chown -R "$SERVICE_USER:$SERVICE_USER" "$INSTALL_DIR"
|
chown -R "$SERVICE_USER:$SERVICE_USER" "$INSTALL_DIR" && echo " [OK] Set ownership for $INSTALL_DIR"
|
||||||
echo " [OK] Set ownership for $INSTALL_DIR"
|
chown -R "$SERVICE_USER:$SERVICE_USER" "$DATA_DIR" && echo " [OK] Set ownership for $DATA_DIR"
|
||||||
chown -R "$SERVICE_USER:$SERVICE_USER" "$DATA_DIR"
|
chown -R "$SERVICE_USER:$SERVICE_USER" "$LOG_DIR" && echo " [OK] Set ownership for $LOG_DIR"
|
||||||
echo " [OK] Set ownership for $DATA_DIR"
|
|
||||||
chown -R "$SERVICE_USER:$SERVICE_USER" "$LOG_DIR"
|
|
||||||
echo " [OK] Set ownership for $LOG_DIR"
|
|
||||||
|
|
||||||
if [[ "$SKIP_SYSTEMD" != true ]]; then
|
if [[ "$SKIP_SYSTEMD" != true ]]; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
echo "STEP 8: Creating Systemd Service"
|
echo "STEP 7: Creating Systemd Service"
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
echo ""
|
echo ""
|
||||||
cat > /etc/systemd/system/myfsio.service << EOF
|
cat > /etc/systemd/system/myfsio.service << EOF
|
||||||
@@ -306,18 +312,16 @@ User=$SERVICE_USER
|
|||||||
Group=$SERVICE_USER
|
Group=$SERVICE_USER
|
||||||
WorkingDirectory=$INSTALL_DIR
|
WorkingDirectory=$INSTALL_DIR
|
||||||
EnvironmentFile=$INSTALL_DIR/myfsio.env
|
EnvironmentFile=$INSTALL_DIR/myfsio.env
|
||||||
ExecStart=$INSTALL_DIR/myfsio
|
ExecStart=$INSTALL_DIR/myfsio serve
|
||||||
Restart=on-failure
|
Restart=on-failure
|
||||||
RestartSec=5
|
RestartSec=5
|
||||||
|
|
||||||
# Security hardening
|
|
||||||
NoNewPrivileges=true
|
NoNewPrivileges=true
|
||||||
ProtectSystem=strict
|
ProtectSystem=strict
|
||||||
ProtectHome=true
|
ProtectHome=true
|
||||||
ReadWritePaths=$DATA_DIR $LOG_DIR
|
ReadWritePaths=$DATA_DIR $LOG_DIR
|
||||||
PrivateTmp=true
|
PrivateTmp=true
|
||||||
|
|
||||||
# Resource limits (adjust as needed)
|
|
||||||
# LimitNOFILE=65535
|
# LimitNOFILE=65535
|
||||||
# MemoryMax=2G
|
# MemoryMax=2G
|
||||||
|
|
||||||
@@ -331,7 +335,7 @@ EOF
|
|||||||
else
|
else
|
||||||
echo ""
|
echo ""
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
echo "STEP 8: Skipping Systemd Service (--no-systemd flag used)"
|
echo "STEP 7: Skipping Systemd Service (--no-systemd flag used)"
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -343,30 +347,33 @@ echo ""
|
|||||||
|
|
||||||
if [[ "$SKIP_SYSTEMD" != true ]]; then
|
if [[ "$SKIP_SYSTEMD" != true ]]; then
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
echo "STEP 9: Start the Service"
|
echo "STEP 8: Start the Service"
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [[ "$AUTO_YES" != true ]]; then
|
if [[ "$AUTO_YES" != true ]]; then
|
||||||
read -p "Would you like to start MyFSIO now? [Y/n] " -n 1 -r
|
read -p "Would you like to start MyFSIO now? [Y/n] " -n 1 -r
|
||||||
echo
|
echo
|
||||||
START_SERVICE=true
|
START_SERVICE=true
|
||||||
if [[ $REPLY =~ ^[Nn]$ ]]; then
|
[[ $REPLY =~ ^[Nn]$ ]] && START_SERVICE=false
|
||||||
START_SERVICE=false
|
|
||||||
fi
|
|
||||||
else
|
else
|
||||||
START_SERVICE=true
|
START_SERVICE=true
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "$START_SERVICE" == true ]]; then
|
if [[ "$START_SERVICE" == true ]]; then
|
||||||
echo " Starting MyFSIO service..."
|
echo " Starting MyFSIO service..."
|
||||||
systemctl start myfsio
|
systemctl start myfsio
|
||||||
echo " [OK] Service started"
|
echo " [OK] Service started"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
read -p "Would you like to enable MyFSIO to start on boot? [Y/n] " -n 1 -r
|
if [[ "$AUTO_YES" != true ]]; then
|
||||||
echo
|
read -p "Would you like to enable MyFSIO to start on boot? [Y/n] " -n 1 -r
|
||||||
if [[ ! $REPLY =~ ^[Nn]$ ]]; then
|
echo
|
||||||
|
if [[ ! $REPLY =~ ^[Nn]$ ]]; then
|
||||||
|
systemctl enable myfsio
|
||||||
|
echo " [OK] Service enabled on boot"
|
||||||
|
fi
|
||||||
|
else
|
||||||
systemctl enable myfsio
|
systemctl enable myfsio
|
||||||
echo " [OK] Service enabled on boot"
|
echo " [OK] Service enabled on boot"
|
||||||
fi
|
fi
|
||||||
@@ -383,21 +390,18 @@ if [[ "$SKIP_SYSTEMD" != true ]]; then
|
|||||||
echo " ============================================"
|
echo " ============================================"
|
||||||
echo " ADMIN CREDENTIALS (save these securely!)"
|
echo " ADMIN CREDENTIALS (save these securely!)"
|
||||||
echo " ============================================"
|
echo " ============================================"
|
||||||
CRED_OUTPUT=$(journalctl -u myfsio --no-pager -n 50 2>/dev/null | grep -A 5 "FIRST RUN - ADMIN CREDENTIALS")
|
CRED_OUTPUT=$(journalctl -u myfsio --no-pager -n 100 2>/dev/null | grep -A 5 "FIRST RUN - ADMIN CREDENTIALS")
|
||||||
ACCESS_KEY=$(echo "$CRED_OUTPUT" | grep "Access Key:" | head -1 | sed 's/.*Access Key: //' | awk '{print $1}')
|
ACCESS_KEY=$(echo "$CRED_OUTPUT" | grep "Access Key:" | head -1 | sed 's/.*Access Key: //' | awk '{print $1}')
|
||||||
SECRET_KEY=$(echo "$CRED_OUTPUT" | grep "Secret Key:" | head -1 | sed 's/.*Secret Key: //' | awk '{print $1}')
|
SECRET_KEY=$(echo "$CRED_OUTPUT" | grep "Secret Key:" | head -1 | sed 's/.*Secret Key: //' | awk '{print $1}')
|
||||||
if [[ -n "$ACCESS_KEY" && "$ACCESS_KEY" != *"from"* && -n "$SECRET_KEY" && "$SECRET_KEY" != *"from"* ]]; then
|
if [[ -n "$ACCESS_KEY" && -n "$SECRET_KEY" ]]; then
|
||||||
echo " Access Key: $ACCESS_KEY"
|
echo " Access Key: $ACCESS_KEY"
|
||||||
echo " Secret Key: $SECRET_KEY"
|
echo " Secret Key: $SECRET_KEY"
|
||||||
else
|
else
|
||||||
echo " [!] Could not extract credentials from service logs."
|
echo " [!] Could not extract credentials from service logs."
|
||||||
echo " Check startup output: journalctl -u myfsio --no-pager | grep -A 5 'ADMIN CREDENTIALS'"
|
echo " Check: journalctl -u myfsio --no-pager | grep -A 5 'ADMIN CREDENTIALS'"
|
||||||
echo " Or reset credentials: $INSTALL_DIR/myfsio reset-cred"
|
echo " Or reset: $INSTALL_DIR/myfsio --reset-cred"
|
||||||
fi
|
fi
|
||||||
echo " ============================================"
|
echo " ============================================"
|
||||||
echo ""
|
|
||||||
echo " NOTE: The IAM config file is encrypted at rest."
|
|
||||||
echo " Credentials are only shown on first run or after reset."
|
|
||||||
else
|
else
|
||||||
echo " [WARNING] MyFSIO may not have started correctly"
|
echo " [WARNING] MyFSIO may not have started correctly"
|
||||||
echo " Check logs with: journalctl -u myfsio -f"
|
echo " Check logs with: journalctl -u myfsio -f"
|
||||||
@@ -405,27 +409,21 @@ if [[ "$SKIP_SYSTEMD" != true ]]; then
|
|||||||
else
|
else
|
||||||
echo " [SKIPPED] Service not started"
|
echo " [SKIPPED] Service not started"
|
||||||
echo ""
|
echo ""
|
||||||
echo " To start manually, run:"
|
echo " Start manually: sudo systemctl start myfsio"
|
||||||
echo " sudo systemctl start myfsio"
|
echo " Enable on boot: sudo systemctl enable myfsio"
|
||||||
echo ""
|
|
||||||
echo " To enable on boot, run:"
|
|
||||||
echo " sudo systemctl enable myfsio"
|
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
HOST_IP=$(hostname -I 2>/dev/null | awk '{print $1}' || echo "localhost")
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "============================================================"
|
echo "============================================================"
|
||||||
echo " Summary"
|
echo " Summary"
|
||||||
echo "============================================================"
|
echo "============================================================"
|
||||||
echo ""
|
echo ""
|
||||||
echo "Access Points:"
|
echo "Access Points:"
|
||||||
echo " API: http://$(hostname -I 2>/dev/null | awk '{print $1}' || echo "localhost"):$API_PORT"
|
echo " S3 API: http://$HOST_IP:$API_PORT"
|
||||||
echo " UI: http://$(hostname -I 2>/dev/null | awk '{print $1}' || echo "localhost"):$UI_PORT/ui"
|
echo " Web UI: http://$HOST_IP:$UI_PORT/ui"
|
||||||
echo ""
|
|
||||||
echo "Credentials:"
|
|
||||||
echo " Admin credentials are shown on first service start (see above)."
|
|
||||||
echo " The IAM config is encrypted at rest and cannot be read directly."
|
|
||||||
echo " To reset credentials: $INSTALL_DIR/myfsio reset-cred"
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "Configuration Files:"
|
echo "Configuration Files:"
|
||||||
echo " Environment: $INSTALL_DIR/myfsio.env"
|
echo " Environment: $INSTALL_DIR/myfsio.env"
|
||||||
@@ -433,18 +431,14 @@ echo " IAM Users: $DATA_DIR/.myfsio.sys/config/iam.json (encrypted)"
|
|||||||
echo " Bucket Policies: $DATA_DIR/.myfsio.sys/config/bucket_policies.json"
|
echo " Bucket Policies: $DATA_DIR/.myfsio.sys/config/bucket_policies.json"
|
||||||
echo " Secret Key: $DATA_DIR/.myfsio.sys/config/.secret (auto-generated)"
|
echo " Secret Key: $DATA_DIR/.myfsio.sys/config/.secret (auto-generated)"
|
||||||
echo ""
|
echo ""
|
||||||
echo "Security Notes:"
|
|
||||||
echo " - Rate limiting is enabled by default (200 req/min)"
|
|
||||||
echo " - Brute-force protection: 5 attempts, 15 min lockout"
|
|
||||||
echo " - Set CORS_ORIGINS to specific domains in production"
|
|
||||||
echo " - Set NUM_TRUSTED_PROXIES if behind a reverse proxy"
|
|
||||||
echo ""
|
|
||||||
echo "Useful Commands:"
|
echo "Useful Commands:"
|
||||||
echo " Check status: sudo systemctl status myfsio"
|
echo " Check status: sudo systemctl status myfsio"
|
||||||
echo " View logs: sudo journalctl -u myfsio -f"
|
echo " View logs: sudo journalctl -u myfsio -f"
|
||||||
echo " Validate config: $INSTALL_DIR/myfsio --check-config"
|
echo " Validate config: $INSTALL_DIR/myfsio --check-config"
|
||||||
echo " Restart: sudo systemctl restart myfsio"
|
echo " Show config: $INSTALL_DIR/myfsio --show-config"
|
||||||
echo " Stop: sudo systemctl stop myfsio"
|
echo " Reset admin: sudo -u $SERVICE_USER $INSTALL_DIR/myfsio --reset-cred"
|
||||||
|
echo " Restart: sudo systemctl restart myfsio"
|
||||||
|
echo " Stop: sudo systemctl stop myfsio"
|
||||||
echo ""
|
echo ""
|
||||||
echo "Documentation: https://go.jzwsite.com/myfsio"
|
echo "Documentation: https://go.jzwsite.com/myfsio"
|
||||||
echo ""
|
echo ""
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
# --data-dir DIR Data directory (default: /var/lib/myfsio)
|
# --data-dir DIR Data directory (default: /var/lib/myfsio)
|
||||||
# --log-dir DIR Log directory (default: /var/log/myfsio)
|
# --log-dir DIR Log directory (default: /var/log/myfsio)
|
||||||
# --user USER System user (default: myfsio)
|
# --user USER System user (default: myfsio)
|
||||||
|
# --no-systemd Skip systemd service teardown
|
||||||
# -y, --yes Skip confirmation prompts
|
# -y, --yes Skip confirmation prompts
|
||||||
#
|
#
|
||||||
|
|
||||||
@@ -24,46 +25,21 @@ LOG_DIR="/var/log/myfsio"
|
|||||||
SERVICE_USER="myfsio"
|
SERVICE_USER="myfsio"
|
||||||
KEEP_DATA=false
|
KEEP_DATA=false
|
||||||
KEEP_LOGS=false
|
KEEP_LOGS=false
|
||||||
|
SKIP_SYSTEMD=false
|
||||||
AUTO_YES=false
|
AUTO_YES=false
|
||||||
|
|
||||||
while [[ $# -gt 0 ]]; do
|
while [[ $# -gt 0 ]]; do
|
||||||
case $1 in
|
case $1 in
|
||||||
--keep-data)
|
--keep-data) KEEP_DATA=true; shift ;;
|
||||||
KEEP_DATA=true
|
--keep-logs) KEEP_LOGS=true; shift ;;
|
||||||
shift
|
--install-dir) INSTALL_DIR="$2"; shift 2 ;;
|
||||||
;;
|
--data-dir) DATA_DIR="$2"; shift 2 ;;
|
||||||
--keep-logs)
|
--log-dir) LOG_DIR="$2"; shift 2 ;;
|
||||||
KEEP_LOGS=true
|
--user) SERVICE_USER="$2"; shift 2 ;;
|
||||||
shift
|
--no-systemd) SKIP_SYSTEMD=true; shift ;;
|
||||||
;;
|
-y|--yes) AUTO_YES=true; shift ;;
|
||||||
--install-dir)
|
-h|--help) head -21 "$0" | tail -16; exit 0 ;;
|
||||||
INSTALL_DIR="$2"
|
*) echo "Unknown option: $1"; exit 1 ;;
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--data-dir)
|
|
||||||
DATA_DIR="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--log-dir)
|
|
||||||
LOG_DIR="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--user)
|
|
||||||
SERVICE_USER="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
-y|--yes)
|
|
||||||
AUTO_YES=true
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
-h|--help)
|
|
||||||
head -20 "$0" | tail -15
|
|
||||||
exit 0
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "Unknown option: $1"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
@@ -125,42 +101,51 @@ if [[ "$AUTO_YES" != true ]]; then
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo ""
|
if [[ "$SKIP_SYSTEMD" != true ]]; then
|
||||||
echo "------------------------------------------------------------"
|
echo ""
|
||||||
echo "STEP 2: Stopping Service"
|
echo "------------------------------------------------------------"
|
||||||
echo "------------------------------------------------------------"
|
echo "STEP 2: Stopping Service"
|
||||||
echo ""
|
echo "------------------------------------------------------------"
|
||||||
if systemctl is-active --quiet myfsio 2>/dev/null; then
|
echo ""
|
||||||
systemctl stop myfsio
|
if systemctl is-active --quiet myfsio 2>/dev/null; then
|
||||||
echo " [OK] Stopped myfsio service"
|
systemctl stop myfsio
|
||||||
else
|
echo " [OK] Stopped myfsio service"
|
||||||
echo " [SKIP] Service not running"
|
else
|
||||||
fi
|
echo " [SKIP] Service not running"
|
||||||
|
fi
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
echo "STEP 3: Disabling Service"
|
echo "STEP 3: Disabling Service"
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
echo ""
|
echo ""
|
||||||
if systemctl is-enabled --quiet myfsio 2>/dev/null; then
|
if systemctl is-enabled --quiet myfsio 2>/dev/null; then
|
||||||
systemctl disable myfsio
|
systemctl disable myfsio
|
||||||
echo " [OK] Disabled myfsio service"
|
echo " [OK] Disabled myfsio service"
|
||||||
else
|
else
|
||||||
echo " [SKIP] Service not enabled"
|
echo " [SKIP] Service not enabled"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
echo "STEP 4: Removing Systemd Service File"
|
echo "STEP 4: Removing Systemd Service File"
|
||||||
echo "------------------------------------------------------------"
|
echo "------------------------------------------------------------"
|
||||||
echo ""
|
echo ""
|
||||||
if [[ -f /etc/systemd/system/myfsio.service ]]; then
|
if [[ -f /etc/systemd/system/myfsio.service ]]; then
|
||||||
rm -f /etc/systemd/system/myfsio.service
|
rm -f /etc/systemd/system/myfsio.service
|
||||||
systemctl daemon-reload
|
systemctl daemon-reload
|
||||||
echo " [OK] Removed /etc/systemd/system/myfsio.service"
|
echo " [OK] Removed /etc/systemd/system/myfsio.service"
|
||||||
echo " [OK] Reloaded systemd daemon"
|
echo " [OK] Reloaded systemd daemon"
|
||||||
|
else
|
||||||
|
echo " [SKIP] Service file not found"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
echo " [SKIP] Service file not found"
|
echo ""
|
||||||
|
echo "------------------------------------------------------------"
|
||||||
|
echo "STEPS 2-4: Skipping Systemd Teardown (--no-systemd)"
|
||||||
|
echo "------------------------------------------------------------"
|
||||||
|
echo ""
|
||||||
|
echo " Stop any running myfsio process manually before continuing."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
@@ -235,11 +220,11 @@ if [[ "$KEEP_DATA" == true ]]; then
|
|||||||
echo " - Secret key: $DATA_DIR/.myfsio.sys/config/.secret"
|
echo " - Secret key: $DATA_DIR/.myfsio.sys/config/.secret"
|
||||||
echo " - Encryption keys: $DATA_DIR/.myfsio.sys/keys/ (if encryption was enabled)"
|
echo " - Encryption keys: $DATA_DIR/.myfsio.sys/keys/ (if encryption was enabled)"
|
||||||
echo ""
|
echo ""
|
||||||
echo "NOTE: The IAM config is encrypted and requires the SECRET_KEY to read."
|
echo "NOTE: The IAM config is encrypted at rest and is unlocked by the .secret"
|
||||||
echo " Keep the .secret file intact for reinstallation."
|
echo " file in the data directory. Keep that file intact for reinstallation."
|
||||||
echo ""
|
echo ""
|
||||||
echo "To reinstall MyFSIO with existing data:"
|
echo "To reinstall MyFSIO with existing data:"
|
||||||
echo " ./install.sh --data-dir $DATA_DIR"
|
echo " ./install.sh --binary ./myfsio --data-dir $DATA_DIR"
|
||||||
echo ""
|
echo ""
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user