7 Commits

Author SHA1 Message Date
f2df64479c Fix S3 versioning (live-object VersionId, DM PUT/DELETE), harden DeleteObjects/ListObjects conformance, and run hot paths on blocking threads 2026-04-23 22:40:38 +08:00
bd405cc2fe Fix S3 versioning/delete markers, path-safety leaks, and error-code conformance; parallelize DeleteObjects; restore per-op rate limits 2026-04-23 20:23:11 +08:00
7ef3820f6e Fix SigV4/SHA256/TCP_NODELAY critical paths; tighten multipart, copy, versioning, and S3 error conformance 2026-04-23 17:52:30 +08:00
e1fb225034 csrf fixes 2026-04-22 23:01:32 +08:00
2767e7e79d Optimize bucket listing for 10K-100K objects
- Shallow listing: read per-directory _index.json once for eTags instead
  of N serial .meta.json reads. Validate prefix for path traversal and
  verify normalized target stays within bucket root.
- Recursive listing: cache full per-directory index during the walk so
  each _index.json is parsed at most once per call.
- Per-bucket listing cache with 5s TTL and per-bucket rebuild mutex.
  Invalidated on put/delete/copy/metadata/tags/multipart-complete.
  Pagination uses partition_point for O(log n) start lookup.
- UI stream endpoint now actually streams via mpsc + Body::from_stream
  instead of buffering into a Vec<String>. Cancels producer on client
  disconnect.
- UI JSON endpoint honors delimiter=/ and returns common_prefixes.
- run_blocking wrapper dispatches sync filesystem work via
  block_in_place on multi-threaded runtimes, falls back to inline on
  current-thread runtimes (unit tests).
2026-04-22 19:55:44 +08:00
217af6d1c6 Full migration and transition to Rust; Remove python artifacts 2026-04-22 17:19:19 +08:00
51d54b42ac Rust fixes 2026-04-22 15:41:18 +08:00
206 changed files with 4699 additions and 55310 deletions

View File

@@ -3,7 +3,7 @@
logs
data
tmp
myfsio-engine/target
myfsio-engine/tests
target
crates/*/tests
Dockerfile
.dockerignore

6
.gitignore vendored
View File

@@ -26,12 +26,8 @@ dist/
*.egg-info/
.eggs/
# Rust / maturin build artifacts
python/myfsio_core/target/
python/myfsio_core/Cargo.lock
# Rust engine build artifacts
rust/myfsio-engine/target/
target/
# Local runtime artifacts
logs/

View File

@@ -2542,6 +2542,15 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "matchers"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9"
dependencies = [
"regex-automata",
]
[[package]]
name = "matchit"
version = "0.8.4"
@@ -2630,7 +2639,7 @@ dependencies = [
[[package]]
name = "myfsio-auth"
version = "0.4.3"
version = "0.4.4"
dependencies = [
"aes",
"base64",
@@ -2655,7 +2664,7 @@ dependencies = [
[[package]]
name = "myfsio-common"
version = "0.4.3"
version = "0.4.4"
dependencies = [
"chrono",
"serde",
@@ -2666,7 +2675,7 @@ dependencies = [
[[package]]
name = "myfsio-crypto"
version = "0.4.3"
version = "0.4.4"
dependencies = [
"aes-gcm",
"base64",
@@ -2687,7 +2696,7 @@ dependencies = [
[[package]]
name = "myfsio-server"
version = "0.4.3"
version = "0.4.4"
dependencies = [
"aes-gcm",
"async-trait",
@@ -2705,6 +2714,8 @@ dependencies = [
"dotenvy",
"duckdb",
"futures",
"hex",
"http-body 1.0.1",
"http-body-util",
"hyper 1.9.0",
"md-5 0.10.6",
@@ -2724,12 +2735,14 @@ dependencies = [
"roxmltree",
"serde",
"serde_json",
"serde_urlencoded",
"sha2 0.10.9",
"subtle",
"sysinfo",
"tempfile",
"tera",
"tokio",
"tokio-stream",
"tokio-util",
"tower",
"tower-http",
@@ -2740,7 +2753,7 @@ dependencies = [
[[package]]
name = "myfsio-storage"
version = "0.4.3"
version = "0.4.4"
dependencies = [
"chrono",
"dashmap",
@@ -2763,10 +2776,11 @@ dependencies = [
[[package]]
name = "myfsio-xml"
version = "0.4.3"
version = "0.4.4"
dependencies = [
"chrono",
"myfsio-common",
"percent-encoding",
"quick-xml",
"serde",
]
@@ -4183,6 +4197,17 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"
dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-util"
version = "0.7.18"
@@ -4333,10 +4358,14 @@ version = "0.3.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"
dependencies = [
"matchers",
"nu-ansi-term",
"once_cell",
"regex-automata",
"sharded-slab",
"smallvec",
"thread_local",
"tracing",
"tracing-core",
"tracing-log",
]

View File

@@ -10,14 +10,14 @@ members = [
]
[workspace.package]
version = "0.4.3"
version = "0.4.4"
edition = "2021"
[workspace.dependencies]
tokio = { version = "1", features = ["full"] }
axum = { version = "0.8" }
tower = { version = "0.5" }
tower-http = { version = "0.6", features = ["cors", "trace", "fs", "compression-gzip"] }
tower-http = { version = "0.6", features = ["cors", "trace", "fs", "compression-gzip", "timeout"] }
hyper = { version = "1" }
bytes = "1"
serde = { version = "1", features = ["derive"] }
@@ -38,11 +38,12 @@ percent-encoding = "2"
regex = "1"
unicode-normalization = "0.1"
tracing = "0.1"
tracing-subscriber = "0.3"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
thiserror = "2"
chrono = { version = "0.4", features = ["serde"] }
base64 = "0.22"
tokio-util = { version = "0.7", features = ["io"] }
tokio-stream = "0.1"
futures = "0.3"
dashmap = "6"
crc32fast = "1"

View File

@@ -6,10 +6,10 @@ RUN apt-get update \
&& apt-get install -y --no-install-recommends build-essential pkg-config libssl-dev \
&& rm -rf /var/lib/apt/lists/*
COPY myfsio-engine ./myfsio-engine
COPY Cargo.toml Cargo.lock ./
COPY crates ./crates
RUN cd myfsio-engine \
&& cargo build --release --bin myfsio-server \
RUN cargo build --release --bin myfsio-server \
&& strip target/release/myfsio-server
@@ -24,9 +24,9 @@ RUN apt-get update \
&& useradd -m -u 1000 myfsio \
&& chown -R myfsio:myfsio /app
COPY --from=builder /build/myfsio-engine/target/release/myfsio-server /usr/local/bin/myfsio-server
COPY --from=builder /build/myfsio-engine/crates/myfsio-server/templates /app/templates
COPY --from=builder /build/myfsio-engine/crates/myfsio-server/static /app/static
COPY --from=builder /build/target/release/myfsio-server /usr/local/bin/myfsio-server
COPY --from=builder /build/crates/myfsio-server/templates /app/templates
COPY --from=builder /build/crates/myfsio-server/static /app/static
COPY docker-entrypoint.sh /app/docker-entrypoint.sh
RUN chmod +x /app/docker-entrypoint.sh \

View File

@@ -1,8 +1,6 @@
# MyFSIO
MyFSIO is an S3-compatible object storage server with a Rust runtime and a filesystem-backed storage engine. The active server lives under `rust/myfsio-engine` and serves both the S3 API and the built-in web UI from a single process.
The `python/` implementation is deprecated as of 2026-04-21. It remains in the repository for migration reference and legacy tests, but new development and supported runtime usage should target the Rust server.
MyFSIO is an S3-compatible object storage server with a Rust runtime and a filesystem-backed storage engine. The repository root is the Cargo workspace; the server serves both the S3 API and the built-in web UI from a single process.
## Features
@@ -29,7 +27,6 @@ If you want API-only mode, set `UI_ENABLED=false`. There is no separate "UI-only
From the repository root:
```bash
cd rust/myfsio-engine
cargo run -p myfsio-server --
```
@@ -60,14 +57,13 @@ UI_ENABLED=false cargo run -p myfsio-server --
## Building a Binary
```bash
cd rust/myfsio-engine
cargo build --release -p myfsio-server
```
Binary locations:
- Linux/macOS: `rust/myfsio-engine/target/release/myfsio-server`
- Windows: `rust/myfsio-engine/target/release/myfsio-server.exe`
- Linux/macOS: `target/release/myfsio-server`
- Windows: `target/release/myfsio-server.exe`
Run the built binary directly:
@@ -166,10 +162,10 @@ data/
## Docker
Build the Rust image from the `rust/` directory:
Build the Rust image from the repository root:
```bash
docker build -t myfsio ./rust
docker build -t myfsio .
docker run --rm -p 5000:5000 -p 5100:5100 -v "${PWD}/data:/app/data" myfsio
```
@@ -180,11 +176,9 @@ If the instance sits behind a reverse proxy, set `API_BASE_URL` to the public S3
The repository includes `scripts/install.sh` for systemd-style Linux installs. Build the Rust binary first, then pass it to the installer:
```bash
cd rust/myfsio-engine
cargo build --release -p myfsio-server
cd ../..
sudo ./scripts/install.sh --binary ./rust/myfsio-engine/target/release/myfsio-server
sudo ./scripts/install.sh --binary ./target/release/myfsio-server
```
The installer copies the binary into `/opt/myfsio/myfsio`, writes `/opt/myfsio/myfsio.env`, and can register a `myfsio.service` unit.
@@ -194,7 +188,6 @@ The installer copies the binary into `/opt/myfsio/myfsio`, writes `/opt/myfsio/m
Run the Rust test suite from the workspace:
```bash
cd rust/myfsio-engine
cargo test
```
@@ -209,4 +202,4 @@ cargo test
}
```
The `version` field comes from the Rust crate version in `rust/myfsio-engine/crates/myfsio-server/Cargo.toml`.
The `version` field comes from the Rust crate version in `crates/myfsio-server/Cargo.toml`.

View File

@@ -8,6 +8,7 @@ pub const STATS_FILE: &str = "stats.json";
pub const ETAG_INDEX_FILE: &str = "etag_index.json";
pub const INDEX_FILE: &str = "_index.json";
pub const MANIFEST_FILE: &str = "manifest.json";
pub const DIR_MARKER_FILE: &str = ".__myfsio_dirobj__";
pub const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"];

View File

@@ -5,13 +5,17 @@ pub enum S3ErrorCode {
AccessDenied,
BadDigest,
BucketAlreadyExists,
BucketAlreadyOwnedByYou,
BucketNotEmpty,
EntityTooLarge,
EntityTooSmall,
InternalError,
InvalidAccessKeyId,
InvalidArgument,
InvalidBucketName,
InvalidKey,
InvalidPart,
InvalidPartOrder,
InvalidPolicyDocument,
InvalidRange,
InvalidRequest,
@@ -19,13 +23,17 @@ pub enum S3ErrorCode {
MalformedXML,
MethodNotAllowed,
NoSuchBucket,
NoSuchBucketPolicy,
NoSuchKey,
NoSuchLifecycleConfiguration,
NoSuchUpload,
NoSuchVersion,
NoSuchTagSet,
PreconditionFailed,
NotModified,
QuotaExceeded,
RequestTimeTooSkewed,
ServerSideEncryptionConfigurationNotFoundError,
SignatureDoesNotMatch,
SlowDown,
}
@@ -36,13 +44,17 @@ impl S3ErrorCode {
Self::AccessDenied => 403,
Self::BadDigest => 400,
Self::BucketAlreadyExists => 409,
Self::BucketAlreadyOwnedByYou => 409,
Self::BucketNotEmpty => 409,
Self::EntityTooLarge => 413,
Self::EntityTooSmall => 400,
Self::InternalError => 500,
Self::InvalidAccessKeyId => 403,
Self::InvalidArgument => 400,
Self::InvalidBucketName => 400,
Self::InvalidKey => 400,
Self::InvalidPart => 400,
Self::InvalidPartOrder => 400,
Self::InvalidPolicyDocument => 400,
Self::InvalidRange => 416,
Self::InvalidRequest => 400,
@@ -50,15 +62,19 @@ impl S3ErrorCode {
Self::MalformedXML => 400,
Self::MethodNotAllowed => 405,
Self::NoSuchBucket => 404,
Self::NoSuchBucketPolicy => 404,
Self::NoSuchKey => 404,
Self::NoSuchLifecycleConfiguration => 404,
Self::NoSuchUpload => 404,
Self::NoSuchVersion => 404,
Self::NoSuchTagSet => 404,
Self::PreconditionFailed => 412,
Self::NotModified => 304,
Self::QuotaExceeded => 403,
Self::RequestTimeTooSkewed => 403,
Self::ServerSideEncryptionConfigurationNotFoundError => 404,
Self::SignatureDoesNotMatch => 403,
Self::SlowDown => 429,
Self::SlowDown => 503,
}
}
@@ -67,13 +83,17 @@ impl S3ErrorCode {
Self::AccessDenied => "AccessDenied",
Self::BadDigest => "BadDigest",
Self::BucketAlreadyExists => "BucketAlreadyExists",
Self::BucketAlreadyOwnedByYou => "BucketAlreadyOwnedByYou",
Self::BucketNotEmpty => "BucketNotEmpty",
Self::EntityTooLarge => "EntityTooLarge",
Self::EntityTooSmall => "EntityTooSmall",
Self::InternalError => "InternalError",
Self::InvalidAccessKeyId => "InvalidAccessKeyId",
Self::InvalidArgument => "InvalidArgument",
Self::InvalidBucketName => "InvalidBucketName",
Self::InvalidKey => "InvalidKey",
Self::InvalidPart => "InvalidPart",
Self::InvalidPartOrder => "InvalidPartOrder",
Self::InvalidPolicyDocument => "InvalidPolicyDocument",
Self::InvalidRange => "InvalidRange",
Self::InvalidRequest => "InvalidRequest",
@@ -81,13 +101,19 @@ impl S3ErrorCode {
Self::MalformedXML => "MalformedXML",
Self::MethodNotAllowed => "MethodNotAllowed",
Self::NoSuchBucket => "NoSuchBucket",
Self::NoSuchBucketPolicy => "NoSuchBucketPolicy",
Self::NoSuchKey => "NoSuchKey",
Self::NoSuchLifecycleConfiguration => "NoSuchLifecycleConfiguration",
Self::NoSuchUpload => "NoSuchUpload",
Self::NoSuchVersion => "NoSuchVersion",
Self::NoSuchTagSet => "NoSuchTagSet",
Self::PreconditionFailed => "PreconditionFailed",
Self::NotModified => "NotModified",
Self::QuotaExceeded => "QuotaExceeded",
Self::RequestTimeTooSkewed => "RequestTimeTooSkewed",
Self::ServerSideEncryptionConfigurationNotFoundError => {
"ServerSideEncryptionConfigurationNotFoundError"
}
Self::SignatureDoesNotMatch => "SignatureDoesNotMatch",
Self::SlowDown => "SlowDown",
}
@@ -98,13 +124,17 @@ impl S3ErrorCode {
Self::AccessDenied => "Access Denied",
Self::BadDigest => "The Content-MD5 or checksum value you specified did not match what we received",
Self::BucketAlreadyExists => "The requested bucket name is not available",
Self::BucketAlreadyOwnedByYou => "Your previous request to create the named bucket succeeded and you already own it",
Self::BucketNotEmpty => "The bucket you tried to delete is not empty",
Self::EntityTooLarge => "Your proposed upload exceeds the maximum allowed size",
Self::EntityTooSmall => "Your proposed upload is smaller than the minimum allowed object size",
Self::InternalError => "We encountered an internal error. Please try again.",
Self::InvalidAccessKeyId => "The access key ID you provided does not exist",
Self::InvalidArgument => "Invalid argument",
Self::InvalidBucketName => "The specified bucket is not valid",
Self::InvalidKey => "The specified key is not valid",
Self::InvalidPart => "One or more of the specified parts could not be found",
Self::InvalidPartOrder => "The list of parts was not in ascending order",
Self::InvalidPolicyDocument => "The content of the form does not meet the conditions specified in the policy document",
Self::InvalidRange => "The requested range is not satisfiable",
Self::InvalidRequest => "Invalid request",
@@ -112,13 +142,17 @@ impl S3ErrorCode {
Self::MalformedXML => "The XML you provided was not well-formed",
Self::MethodNotAllowed => "The specified method is not allowed against this resource",
Self::NoSuchBucket => "The specified bucket does not exist",
Self::NoSuchBucketPolicy => "The bucket policy does not exist",
Self::NoSuchKey => "The specified key does not exist",
Self::NoSuchLifecycleConfiguration => "The lifecycle configuration does not exist",
Self::NoSuchUpload => "The specified multipart upload does not exist",
Self::NoSuchVersion => "The specified version does not exist",
Self::NoSuchTagSet => "The TagSet does not exist",
Self::PreconditionFailed => "At least one of the preconditions you specified did not hold",
Self::NotModified => "Not Modified",
Self::QuotaExceeded => "The bucket quota has been exceeded",
Self::RequestTimeTooSkewed => "The difference between the request time and the server's time is too large",
Self::ServerSideEncryptionConfigurationNotFoundError => "The server side encryption configuration was not found",
Self::SignatureDoesNotMatch => "The request signature we calculated does not match the signature you provided",
Self::SlowDown => "Please reduce your request rate",
}

View File

@@ -12,6 +12,10 @@ pub struct ObjectMeta {
pub content_type: Option<String>,
pub storage_class: Option<String>,
pub metadata: HashMap<String, String>,
#[serde(default)]
pub version_id: Option<String>,
#[serde(default)]
pub is_delete_marker: bool,
}
impl ObjectMeta {
@@ -24,10 +28,19 @@ impl ObjectMeta {
content_type: None,
storage_class: Some("STANDARD".to_string()),
metadata: HashMap::new(),
version_id: None,
is_delete_marker: false,
}
}
}
#[derive(Debug, Clone, Default)]
pub struct DeleteOutcome {
pub version_id: Option<String>,
pub is_delete_marker: bool,
pub existed: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BucketMeta {
pub name: String,

View File

@@ -82,11 +82,35 @@ impl EncryptionMetadata {
pub struct EncryptionService {
master_key: [u8; 32],
kms: Option<std::sync::Arc<KmsService>>,
config: EncryptionConfig,
}
#[derive(Debug, Clone, Copy)]
pub struct EncryptionConfig {
pub chunk_size: usize,
}
impl Default for EncryptionConfig {
fn default() -> Self {
Self { chunk_size: 65_536 }
}
}
impl EncryptionService {
pub fn new(master_key: [u8; 32], kms: Option<std::sync::Arc<KmsService>>) -> Self {
Self { master_key, kms }
Self::with_config(master_key, kms, EncryptionConfig::default())
}
pub fn with_config(
master_key: [u8; 32],
kms: Option<std::sync::Arc<KmsService>>,
config: EncryptionConfig,
) -> Self {
Self {
master_key,
kms,
config,
}
}
pub fn generate_data_key(&self) -> ([u8; 32], [u8; 12]) {
@@ -192,7 +216,10 @@ impl EncryptionService {
let op = output_path.to_owned();
let ak = actual_key;
let n = nonce;
tokio::task::spawn_blocking(move || encrypt_stream_chunked(&ip, &op, &ak, &n, None))
let chunk_size = self.config.chunk_size;
tokio::task::spawn_blocking(move || {
encrypt_stream_chunked(&ip, &op, &ak, &n, Some(chunk_size))
})
.await
.map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))??;

View File

@@ -19,18 +19,22 @@ hyper = { workspace = true }
bytes = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
serde_urlencoded = "0.7"
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
tokio-util = { workspace = true }
tokio-stream = { workspace = true }
chrono = { workspace = true }
uuid = { workspace = true }
futures = { workspace = true }
http-body = "1"
http-body-util = "0.1"
percent-encoding = { workspace = true }
quick-xml = { workspace = true }
mime_guess = "2"
crc32fast = { workspace = true }
sha2 = { workspace = true }
hex = { workspace = true }
duckdb = { workspace = true }
roxmltree = "0.20"
parking_lot = { workspace = true }

View File

@@ -0,0 +1,621 @@
use std::net::SocketAddr;
use std::path::PathBuf;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RateLimitSetting {
pub max_requests: u32,
pub window_seconds: u64,
}
impl RateLimitSetting {
pub const fn new(max_requests: u32, window_seconds: u64) -> Self {
Self {
max_requests,
window_seconds,
}
}
}
#[derive(Debug, Clone)]
pub struct ServerConfig {
pub bind_addr: SocketAddr,
pub ui_bind_addr: SocketAddr,
pub storage_root: PathBuf,
pub region: String,
pub iam_config_path: PathBuf,
pub sigv4_timestamp_tolerance_secs: u64,
pub presigned_url_min_expiry: u64,
pub presigned_url_max_expiry: u64,
pub secret_key: Option<String>,
pub encryption_enabled: bool,
pub encryption_chunk_size_bytes: usize,
pub kms_enabled: bool,
pub kms_generate_data_key_min_bytes: usize,
pub kms_generate_data_key_max_bytes: usize,
pub gc_enabled: bool,
pub gc_interval_hours: f64,
pub gc_temp_file_max_age_hours: f64,
pub gc_multipart_max_age_days: u64,
pub gc_lock_file_max_age_hours: f64,
pub gc_dry_run: bool,
pub integrity_enabled: bool,
pub metrics_enabled: bool,
pub metrics_history_enabled: bool,
pub metrics_interval_minutes: u64,
pub metrics_retention_hours: u64,
pub metrics_history_interval_minutes: u64,
pub metrics_history_retention_hours: u64,
pub lifecycle_enabled: bool,
pub lifecycle_max_history_per_bucket: usize,
pub website_hosting_enabled: bool,
pub object_key_max_length_bytes: usize,
pub object_tag_limit: usize,
pub object_cache_max_size: usize,
pub bucket_config_cache_ttl_seconds: f64,
pub replication_connect_timeout_secs: u64,
pub replication_read_timeout_secs: u64,
pub replication_max_retries: u32,
pub replication_streaming_threshold_bytes: u64,
pub replication_max_failures_per_bucket: usize,
pub site_sync_enabled: bool,
pub site_sync_interval_secs: u64,
pub site_sync_batch_size: usize,
pub site_sync_connect_timeout_secs: u64,
pub site_sync_read_timeout_secs: u64,
pub site_sync_max_retries: u32,
pub site_sync_clock_skew_tolerance: f64,
pub site_id: Option<String>,
pub site_endpoint: Option<String>,
pub site_region: String,
pub site_priority: i32,
pub api_base_url: String,
pub num_trusted_proxies: usize,
pub allowed_redirect_hosts: Vec<String>,
pub allow_internal_endpoints: bool,
pub cors_origins: Vec<String>,
pub cors_methods: Vec<String>,
pub cors_allow_headers: Vec<String>,
pub cors_expose_headers: Vec<String>,
pub session_lifetime_days: u64,
pub log_level: String,
pub multipart_min_part_size: u64,
pub bulk_delete_max_keys: usize,
pub stream_chunk_size: usize,
pub request_body_timeout_secs: u64,
pub ratelimit_default: RateLimitSetting,
pub ratelimit_list_buckets: RateLimitSetting,
pub ratelimit_bucket_ops: RateLimitSetting,
pub ratelimit_object_ops: RateLimitSetting,
pub ratelimit_head_ops: RateLimitSetting,
pub ratelimit_admin: RateLimitSetting,
pub ratelimit_storage_uri: String,
pub ui_enabled: bool,
pub templates_dir: PathBuf,
pub static_dir: PathBuf,
}
impl ServerConfig {
pub fn from_env() -> Self {
let host = std::env::var("HOST").unwrap_or_else(|_| "127.0.0.1".to_string());
let port: u16 = std::env::var("PORT")
.unwrap_or_else(|_| "5000".to_string())
.parse()
.unwrap_or(5000);
let host_ip: std::net::IpAddr = host.parse().unwrap();
let bind_addr = SocketAddr::new(host_ip, port);
let ui_port: u16 = std::env::var("UI_PORT")
.unwrap_or_else(|_| "5100".to_string())
.parse()
.unwrap_or(5100);
let storage_root = std::env::var("STORAGE_ROOT").unwrap_or_else(|_| "./data".to_string());
let region = std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string());
let storage_path = PathBuf::from(&storage_root);
let iam_config_path = std::env::var("IAM_CONFIG")
.map(PathBuf::from)
.unwrap_or_else(|_| {
storage_path
.join(".myfsio.sys")
.join("config")
.join("iam.json")
});
let sigv4_timestamp_tolerance_secs: u64 =
std::env::var("SIGV4_TIMESTAMP_TOLERANCE_SECONDS")
.unwrap_or_else(|_| "900".to_string())
.parse()
.unwrap_or(900);
let presigned_url_min_expiry: u64 = std::env::var("PRESIGNED_URL_MIN_EXPIRY_SECONDS")
.unwrap_or_else(|_| "1".to_string())
.parse()
.unwrap_or(1);
let presigned_url_max_expiry: u64 = std::env::var("PRESIGNED_URL_MAX_EXPIRY_SECONDS")
.unwrap_or_else(|_| "604800".to_string())
.parse()
.unwrap_or(604800);
let secret_key = {
let env_key = std::env::var("SECRET_KEY").ok();
match env_key {
Some(k) if !k.is_empty() && k != "dev-secret-key" => Some(k),
_ => {
let secret_file = storage_path
.join(".myfsio.sys")
.join("config")
.join(".secret");
std::fs::read_to_string(&secret_file)
.ok()
.map(|s| s.trim().to_string())
}
}
};
let encryption_enabled = parse_bool_env("ENCRYPTION_ENABLED", false);
let encryption_chunk_size_bytes = parse_usize_env("ENCRYPTION_CHUNK_SIZE_BYTES", 65_536);
let kms_enabled = parse_bool_env("KMS_ENABLED", false);
let kms_generate_data_key_min_bytes = parse_usize_env("KMS_GENERATE_DATA_KEY_MIN_BYTES", 1);
let kms_generate_data_key_max_bytes =
parse_usize_env("KMS_GENERATE_DATA_KEY_MAX_BYTES", 1024);
let gc_enabled = parse_bool_env("GC_ENABLED", false);
let gc_interval_hours = parse_f64_env("GC_INTERVAL_HOURS", 6.0);
let gc_temp_file_max_age_hours = parse_f64_env("GC_TEMP_FILE_MAX_AGE_HOURS", 24.0);
let gc_multipart_max_age_days = parse_u64_env("GC_MULTIPART_MAX_AGE_DAYS", 7);
let gc_lock_file_max_age_hours = parse_f64_env("GC_LOCK_FILE_MAX_AGE_HOURS", 1.0);
let gc_dry_run = parse_bool_env("GC_DRY_RUN", false);
let integrity_enabled = parse_bool_env("INTEGRITY_ENABLED", false);
let metrics_enabled = parse_bool_env("OPERATION_METRICS_ENABLED", false);
let metrics_history_enabled = parse_bool_env("METRICS_HISTORY_ENABLED", false);
let metrics_interval_minutes = parse_u64_env("OPERATION_METRICS_INTERVAL_MINUTES", 5);
let metrics_retention_hours = parse_u64_env("OPERATION_METRICS_RETENTION_HOURS", 24);
let metrics_history_interval_minutes = parse_u64_env("METRICS_HISTORY_INTERVAL_MINUTES", 5);
let metrics_history_retention_hours = parse_u64_env("METRICS_HISTORY_RETENTION_HOURS", 24);
let lifecycle_enabled = parse_bool_env("LIFECYCLE_ENABLED", false);
let lifecycle_max_history_per_bucket =
parse_usize_env("LIFECYCLE_MAX_HISTORY_PER_BUCKET", 50);
let website_hosting_enabled = parse_bool_env("WEBSITE_HOSTING_ENABLED", false);
let object_key_max_length_bytes = parse_usize_env("OBJECT_KEY_MAX_LENGTH_BYTES", 1024);
let object_tag_limit = parse_usize_env("OBJECT_TAG_LIMIT", 50);
let object_cache_max_size = parse_usize_env("OBJECT_CACHE_MAX_SIZE", 100);
let bucket_config_cache_ttl_seconds =
parse_f64_env("BUCKET_CONFIG_CACHE_TTL_SECONDS", 30.0);
let replication_connect_timeout_secs =
parse_u64_env("REPLICATION_CONNECT_TIMEOUT_SECONDS", 5);
let replication_read_timeout_secs = parse_u64_env("REPLICATION_READ_TIMEOUT_SECONDS", 30);
let replication_max_retries = parse_u64_env("REPLICATION_MAX_RETRIES", 2) as u32;
let replication_streaming_threshold_bytes =
parse_u64_env("REPLICATION_STREAMING_THRESHOLD_BYTES", 10_485_760);
let replication_max_failures_per_bucket =
parse_u64_env("REPLICATION_MAX_FAILURES_PER_BUCKET", 50) as usize;
let site_sync_enabled = parse_bool_env("SITE_SYNC_ENABLED", false);
let site_sync_interval_secs = parse_u64_env("SITE_SYNC_INTERVAL_SECONDS", 60);
let site_sync_batch_size = parse_u64_env("SITE_SYNC_BATCH_SIZE", 100) as usize;
let site_sync_connect_timeout_secs = parse_u64_env("SITE_SYNC_CONNECT_TIMEOUT_SECONDS", 10);
let site_sync_read_timeout_secs = parse_u64_env("SITE_SYNC_READ_TIMEOUT_SECONDS", 120);
let site_sync_max_retries = parse_u64_env("SITE_SYNC_MAX_RETRIES", 2) as u32;
let site_sync_clock_skew_tolerance: f64 =
std::env::var("SITE_SYNC_CLOCK_SKEW_TOLERANCE_SECONDS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(1.0);
let site_id = parse_optional_string_env("SITE_ID");
let site_endpoint = parse_optional_string_env("SITE_ENDPOINT");
let site_region = std::env::var("SITE_REGION").unwrap_or_else(|_| region.clone());
let site_priority = parse_i32_env("SITE_PRIORITY", 100);
let api_base_url = std::env::var("API_BASE_URL")
.unwrap_or_else(|_| format!("http://{}", bind_addr))
.trim_end_matches('/')
.to_string();
let num_trusted_proxies = parse_usize_env("NUM_TRUSTED_PROXIES", 0);
let allowed_redirect_hosts = parse_list_env("ALLOWED_REDIRECT_HOSTS", "");
let allow_internal_endpoints = parse_bool_env("ALLOW_INTERNAL_ENDPOINTS", false);
let cors_origins = parse_list_env("CORS_ORIGINS", "*");
let cors_methods = parse_list_env("CORS_METHODS", "GET,PUT,POST,DELETE,OPTIONS,HEAD");
let cors_allow_headers = parse_list_env("CORS_ALLOW_HEADERS", "*");
let cors_expose_headers = parse_list_env("CORS_EXPOSE_HEADERS", "*");
let session_lifetime_days = parse_u64_env("SESSION_LIFETIME_DAYS", 1);
let log_level = std::env::var("LOG_LEVEL").unwrap_or_else(|_| "INFO".to_string());
let multipart_min_part_size = parse_u64_env("MULTIPART_MIN_PART_SIZE", 5_242_880);
let bulk_delete_max_keys = parse_usize_env("BULK_DELETE_MAX_KEYS", 1000);
let stream_chunk_size = parse_usize_env("STREAM_CHUNK_SIZE", 1_048_576);
let request_body_timeout_secs = parse_u64_env("REQUEST_BODY_TIMEOUT_SECONDS", 60);
let ratelimit_default =
parse_rate_limit_env("RATE_LIMIT_DEFAULT", RateLimitSetting::new(500, 60));
let ratelimit_list_buckets =
parse_rate_limit_env("RATE_LIMIT_LIST_BUCKETS", ratelimit_default);
let ratelimit_bucket_ops =
parse_rate_limit_env("RATE_LIMIT_BUCKET_OPS", ratelimit_default);
let ratelimit_object_ops =
parse_rate_limit_env("RATE_LIMIT_OBJECT_OPS", ratelimit_default);
let ratelimit_head_ops =
parse_rate_limit_env("RATE_LIMIT_HEAD_OPS", ratelimit_default);
let ratelimit_admin =
parse_rate_limit_env("RATE_LIMIT_ADMIN", RateLimitSetting::new(60, 60));
let ratelimit_storage_uri =
std::env::var("RATE_LIMIT_STORAGE_URI").unwrap_or_else(|_| "memory://".to_string());
let ui_enabled = parse_bool_env("UI_ENABLED", true);
let templates_dir = std::env::var("TEMPLATES_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| default_templates_dir());
let static_dir = std::env::var("STATIC_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| default_static_dir());
Self {
bind_addr,
ui_bind_addr: SocketAddr::new(host_ip, ui_port),
storage_root: storage_path,
region,
iam_config_path,
sigv4_timestamp_tolerance_secs,
presigned_url_min_expiry,
presigned_url_max_expiry,
secret_key,
encryption_enabled,
encryption_chunk_size_bytes,
kms_enabled,
kms_generate_data_key_min_bytes,
kms_generate_data_key_max_bytes,
gc_enabled,
gc_interval_hours,
gc_temp_file_max_age_hours,
gc_multipart_max_age_days,
gc_lock_file_max_age_hours,
gc_dry_run,
integrity_enabled,
metrics_enabled,
metrics_history_enabled,
metrics_interval_minutes,
metrics_retention_hours,
metrics_history_interval_minutes,
metrics_history_retention_hours,
lifecycle_enabled,
lifecycle_max_history_per_bucket,
website_hosting_enabled,
object_key_max_length_bytes,
object_tag_limit,
object_cache_max_size,
bucket_config_cache_ttl_seconds,
replication_connect_timeout_secs,
replication_read_timeout_secs,
replication_max_retries,
replication_streaming_threshold_bytes,
replication_max_failures_per_bucket,
site_sync_enabled,
site_sync_interval_secs,
site_sync_batch_size,
site_sync_connect_timeout_secs,
site_sync_read_timeout_secs,
site_sync_max_retries,
site_sync_clock_skew_tolerance,
site_id,
site_endpoint,
site_region,
site_priority,
api_base_url,
num_trusted_proxies,
allowed_redirect_hosts,
allow_internal_endpoints,
cors_origins,
cors_methods,
cors_allow_headers,
cors_expose_headers,
session_lifetime_days,
log_level,
multipart_min_part_size,
bulk_delete_max_keys,
stream_chunk_size,
request_body_timeout_secs,
ratelimit_default,
ratelimit_list_buckets,
ratelimit_bucket_ops,
ratelimit_object_ops,
ratelimit_head_ops,
ratelimit_admin,
ratelimit_storage_uri,
ui_enabled,
templates_dir,
static_dir,
}
}
}
impl Default for ServerConfig {
fn default() -> Self {
Self {
bind_addr: "127.0.0.1:5000".parse().unwrap(),
ui_bind_addr: "127.0.0.1:5100".parse().unwrap(),
storage_root: PathBuf::from("./data"),
region: "us-east-1".to_string(),
iam_config_path: PathBuf::from("./data/.myfsio.sys/config/iam.json"),
sigv4_timestamp_tolerance_secs: 900,
presigned_url_min_expiry: 1,
presigned_url_max_expiry: 604_800,
secret_key: None,
encryption_enabled: false,
encryption_chunk_size_bytes: 65_536,
kms_enabled: false,
kms_generate_data_key_min_bytes: 1,
kms_generate_data_key_max_bytes: 1024,
gc_enabled: false,
gc_interval_hours: 6.0,
gc_temp_file_max_age_hours: 24.0,
gc_multipart_max_age_days: 7,
gc_lock_file_max_age_hours: 1.0,
gc_dry_run: false,
integrity_enabled: false,
metrics_enabled: false,
metrics_history_enabled: false,
metrics_interval_minutes: 5,
metrics_retention_hours: 24,
metrics_history_interval_minutes: 5,
metrics_history_retention_hours: 24,
lifecycle_enabled: false,
lifecycle_max_history_per_bucket: 50,
website_hosting_enabled: false,
object_key_max_length_bytes: 1024,
object_tag_limit: 50,
object_cache_max_size: 100,
bucket_config_cache_ttl_seconds: 30.0,
replication_connect_timeout_secs: 5,
replication_read_timeout_secs: 30,
replication_max_retries: 2,
replication_streaming_threshold_bytes: 10_485_760,
replication_max_failures_per_bucket: 50,
site_sync_enabled: false,
site_sync_interval_secs: 60,
site_sync_batch_size: 100,
site_sync_connect_timeout_secs: 10,
site_sync_read_timeout_secs: 120,
site_sync_max_retries: 2,
site_sync_clock_skew_tolerance: 1.0,
site_id: None,
site_endpoint: None,
site_region: "us-east-1".to_string(),
site_priority: 100,
api_base_url: "http://127.0.0.1:5000".to_string(),
num_trusted_proxies: 0,
allowed_redirect_hosts: Vec::new(),
allow_internal_endpoints: false,
cors_origins: vec!["*".to_string()],
cors_methods: vec![
"GET".to_string(),
"PUT".to_string(),
"POST".to_string(),
"DELETE".to_string(),
"OPTIONS".to_string(),
"HEAD".to_string(),
],
cors_allow_headers: vec!["*".to_string()],
cors_expose_headers: vec!["*".to_string()],
session_lifetime_days: 1,
log_level: "INFO".to_string(),
multipart_min_part_size: 5_242_880,
bulk_delete_max_keys: 1000,
stream_chunk_size: 1_048_576,
request_body_timeout_secs: 60,
ratelimit_default: RateLimitSetting::new(500, 60),
ratelimit_list_buckets: RateLimitSetting::new(500, 60),
ratelimit_bucket_ops: RateLimitSetting::new(500, 60),
ratelimit_object_ops: RateLimitSetting::new(500, 60),
ratelimit_head_ops: RateLimitSetting::new(500, 60),
ratelimit_admin: RateLimitSetting::new(60, 60),
ratelimit_storage_uri: "memory://".to_string(),
ui_enabled: true,
templates_dir: default_templates_dir(),
static_dir: default_static_dir(),
}
}
}
fn default_templates_dir() -> PathBuf {
let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
manifest_dir.join("templates")
}
fn default_static_dir() -> PathBuf {
let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
for candidate in [
manifest_dir.join("static"),
manifest_dir.join("..").join("..").join("..").join("static"),
] {
if candidate.exists() {
return candidate;
}
}
manifest_dir.join("static")
}
fn parse_u64_env(key: &str, default: u64) -> u64 {
std::env::var(key)
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(default)
}
fn parse_usize_env(key: &str, default: usize) -> usize {
std::env::var(key)
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(default)
}
fn parse_i32_env(key: &str, default: i32) -> i32 {
std::env::var(key)
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(default)
}
fn parse_f64_env(key: &str, default: f64) -> f64 {
std::env::var(key)
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(default)
}
fn parse_bool_env(key: &str, default: bool) -> bool {
std::env::var(key)
.ok()
.map(|value| {
matches!(
value.trim().to_ascii_lowercase().as_str(),
"1" | "true" | "yes" | "on"
)
})
.unwrap_or(default)
}
fn parse_optional_string_env(key: &str) -> Option<String> {
std::env::var(key)
.ok()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
}
fn parse_list_env(key: &str, default: &str) -> Vec<String> {
std::env::var(key)
.unwrap_or_else(|_| default.to_string())
.split(',')
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
.collect()
}
pub fn parse_rate_limit(value: &str) -> Option<RateLimitSetting> {
let trimmed = value.trim();
if let Some((requests, window)) = trimmed.split_once('/') {
let max_requests = requests.trim().parse::<u32>().ok()?;
if max_requests == 0 {
return None;
}
let window_str = window.trim().to_ascii_lowercase();
let window_seconds = if let Ok(n) = window_str.parse::<u64>() {
if n == 0 {
return None;
}
n
} else {
match window_str.as_str() {
"s" | "sec" | "second" | "seconds" => 1,
"m" | "min" | "minute" | "minutes" => 60,
"h" | "hr" | "hour" | "hours" => 3600,
"d" | "day" | "days" => 86_400,
_ => return None,
}
};
return Some(RateLimitSetting::new(max_requests, window_seconds));
}
let parts = trimmed.split_whitespace().collect::<Vec<_>>();
if parts.len() != 3 || !parts[1].eq_ignore_ascii_case("per") {
return None;
}
let max_requests = parts[0].parse::<u32>().ok()?;
if max_requests == 0 {
return None;
}
let window_seconds = match parts[2].to_ascii_lowercase().as_str() {
"second" | "seconds" => 1,
"minute" | "minutes" => 60,
"hour" | "hours" => 3600,
"day" | "days" => 86_400,
_ => return None,
};
Some(RateLimitSetting::new(max_requests, window_seconds))
}
fn parse_rate_limit_env(key: &str, default: RateLimitSetting) -> RateLimitSetting {
std::env::var(key)
.ok()
.and_then(|value| parse_rate_limit(&value))
.unwrap_or(default)
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{Mutex, OnceLock};
fn env_lock() -> &'static Mutex<()> {
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
LOCK.get_or_init(|| Mutex::new(()))
}
#[test]
fn parses_rate_limit_text() {
assert_eq!(
parse_rate_limit("200 per minute"),
Some(RateLimitSetting::new(200, 60))
);
assert_eq!(
parse_rate_limit("3 per hours"),
Some(RateLimitSetting::new(3, 3600))
);
assert_eq!(
parse_rate_limit("50000/60"),
Some(RateLimitSetting::new(50000, 60))
);
assert_eq!(
parse_rate_limit("100/minute"),
Some(RateLimitSetting::new(100, 60))
);
assert_eq!(parse_rate_limit("0/60"), None);
assert_eq!(parse_rate_limit("0 per minute"), None);
assert_eq!(parse_rate_limit("bad"), None);
}
#[test]
fn env_defaults_and_invalid_values_fall_back() {
let _guard = env_lock().lock().unwrap();
std::env::remove_var("OBJECT_KEY_MAX_LENGTH_BYTES");
std::env::set_var("OBJECT_TAG_LIMIT", "not-a-number");
std::env::set_var("RATE_LIMIT_DEFAULT", "invalid");
let config = ServerConfig::from_env();
assert_eq!(config.object_key_max_length_bytes, 1024);
assert_eq!(config.object_tag_limit, 50);
assert_eq!(config.ratelimit_default, RateLimitSetting::new(500, 60));
std::env::remove_var("OBJECT_TAG_LIMIT");
std::env::remove_var("RATE_LIMIT_DEFAULT");
}
#[test]
fn env_overrides_new_values() {
let _guard = env_lock().lock().unwrap();
std::env::set_var("OBJECT_KEY_MAX_LENGTH_BYTES", "2048");
std::env::set_var("GC_DRY_RUN", "true");
std::env::set_var("RATE_LIMIT_ADMIN", "7 per second");
std::env::set_var("HOST", "127.0.0.1");
std::env::set_var("PORT", "5501");
std::env::remove_var("API_BASE_URL");
let config = ServerConfig::from_env();
assert_eq!(config.object_key_max_length_bytes, 2048);
assert!(config.gc_dry_run);
assert_eq!(config.ratelimit_admin, RateLimitSetting::new(7, 1));
assert_eq!(config.api_base_url, "http://127.0.0.1:5501");
std::env::remove_var("OBJECT_KEY_MAX_LENGTH_BYTES");
std::env::remove_var("GC_DRY_RUN");
std::env::remove_var("RATE_LIMIT_ADMIN");
std::env::remove_var("HOST");
std::env::remove_var("PORT");
}
}

View File

@@ -218,10 +218,7 @@ pub async fn get_encryption(state: &AppState, bucket: &str) -> Response {
} else {
xml_response(
StatusCode::NOT_FOUND,
S3Error::new(
S3ErrorCode::InvalidRequest,
"The server side encryption configuration was not found",
)
S3Error::from_code(S3ErrorCode::ServerSideEncryptionConfigurationNotFoundError)
.to_xml(),
)
}
@@ -270,11 +267,7 @@ pub async fn get_lifecycle(state: &AppState, bucket: &str) -> Response {
} else {
xml_response(
StatusCode::NOT_FOUND,
S3Error::new(
S3ErrorCode::NoSuchKey,
"The lifecycle configuration does not exist",
)
.to_xml(),
S3Error::from_code(S3ErrorCode::NoSuchLifecycleConfiguration).to_xml(),
)
}
}
@@ -421,7 +414,7 @@ pub async fn get_policy(state: &AppState, bucket: &str) -> Response {
} else {
xml_response(
StatusCode::NOT_FOUND,
S3Error::new(S3ErrorCode::NoSuchKey, "No bucket policy attached").to_xml(),
S3Error::from_code(S3ErrorCode::NoSuchBucketPolicy).to_xml(),
)
}
}
@@ -1095,10 +1088,43 @@ pub async fn list_object_versions(
|| archived_versions.len() > archived_count;
xml.push_str(&format!("<IsTruncated>{}</IsTruncated>", is_truncated));
let current_keys: std::collections::HashSet<String> = objects
.iter()
.take(current_count)
.map(|o| o.key.clone())
.collect();
let mut latest_archived_per_key: std::collections::HashMap<String, String> =
std::collections::HashMap::new();
for v in archived_versions.iter().take(archived_count) {
if current_keys.contains(&v.key) {
continue;
}
let existing = latest_archived_per_key.get(&v.key).cloned();
match existing {
None => {
latest_archived_per_key.insert(v.key.clone(), v.version_id.clone());
}
Some(existing_id) => {
let existing_ts = archived_versions
.iter()
.find(|x| x.key == v.key && x.version_id == existing_id)
.map(|x| x.last_modified)
.unwrap_or(v.last_modified);
if v.last_modified > existing_ts {
latest_archived_per_key.insert(v.key.clone(), v.version_id.clone());
}
}
}
}
for obj in objects.iter().take(current_count) {
let version_id = obj.version_id.clone().unwrap_or_else(|| "null".to_string());
xml.push_str("<Version>");
xml.push_str(&format!("<Key>{}</Key>", xml_escape(&obj.key)));
xml.push_str("<VersionId>null</VersionId>");
xml.push_str(&format!(
"<VersionId>{}</VersionId>",
xml_escape(&version_id)
));
xml.push_str("<IsLatest>true</IsLatest>");
xml.push_str(&format!(
"<LastModified>{}</LastModified>",
@@ -1116,23 +1142,34 @@ pub async fn list_object_versions(
}
for version in archived_versions.iter().take(archived_count) {
xml.push_str("<Version>");
let is_latest = latest_archived_per_key
.get(&version.key)
.map(|id| id == &version.version_id)
.unwrap_or(false);
let tag = if version.is_delete_marker {
"DeleteMarker"
} else {
"Version"
};
xml.push_str(&format!("<{}>", tag));
xml.push_str(&format!("<Key>{}</Key>", xml_escape(&version.key)));
xml.push_str(&format!(
"<VersionId>{}</VersionId>",
xml_escape(&version.version_id)
));
xml.push_str("<IsLatest>false</IsLatest>");
xml.push_str(&format!("<IsLatest>{}</IsLatest>", is_latest));
xml.push_str(&format!(
"<LastModified>{}</LastModified>",
myfsio_xml::response::format_s3_datetime(&version.last_modified)
));
if !version.is_delete_marker {
if let Some(ref etag) = version.etag {
xml.push_str(&format!("<ETag>\"{}\"</ETag>", xml_escape(etag)));
}
xml.push_str(&format!("<Size>{}</Size>", version.size));
xml.push_str("<StorageClass>STANDARD</StorageClass>");
xml.push_str("</Version>");
}
xml.push_str(&format!("</{}>", tag));
}
xml.push_str("</ListVersionsResult>");
@@ -1172,6 +1209,36 @@ pub async fn put_object_tagging(state: &AppState, bucket: &str, key: &str, body:
let xml_str = String::from_utf8_lossy(&body_bytes);
let tags = parse_tagging_xml(&xml_str);
if tags.len() > state.config.object_tag_limit {
return xml_response(
StatusCode::BAD_REQUEST,
S3Error::new(
S3ErrorCode::InvalidTag,
format!("Maximum {} tags allowed", state.config.object_tag_limit),
)
.to_xml(),
);
}
for tag in &tags {
if tag.key.is_empty() || tag.key.len() > 128 {
return xml_response(
StatusCode::BAD_REQUEST,
S3Error::new(S3ErrorCode::InvalidTag, "Tag key length must be 1-128").to_xml(),
);
}
if tag.value.len() > 256 {
return xml_response(
StatusCode::BAD_REQUEST,
S3Error::new(S3ErrorCode::InvalidTag, "Tag value length must be 0-256").to_xml(),
);
}
if tag.key.contains('=') {
return xml_response(
StatusCode::BAD_REQUEST,
S3Error::new(S3ErrorCode::InvalidTag, "Tag keys must not contain '='").to_xml(),
);
}
}
match state.storage.set_object_tags(bucket, key, &tags).await {
Ok(()) => StatusCode::OK.into_response(),

View File

@@ -294,8 +294,17 @@ async fn generate_data_key_inner(state: AppState, body: Body, include_plaintext:
.and_then(|v| v.as_u64())
.unwrap_or(32) as usize;
if !(1..=1024).contains(&num_bytes) {
return json_err(StatusCode::BAD_REQUEST, "NumberOfBytes must be 1-1024");
if num_bytes < state.config.kms_generate_data_key_min_bytes
|| num_bytes > state.config.kms_generate_data_key_max_bytes
{
return json_err(
StatusCode::BAD_REQUEST,
&format!(
"NumberOfBytes must be {}-{}",
state.config.kms_generate_data_key_min_bytes,
state.config.kms_generate_data_key_max_bytes
),
);
}
match kms.generate_data_key(key_id, num_bytes).await {
@@ -389,8 +398,17 @@ pub async fn generate_random(State(state): State<AppState>, body: Body) -> Respo
.and_then(|v| v.as_u64())
.unwrap_or(32) as usize;
if !(1..=1024).contains(&num_bytes) {
return json_err(StatusCode::BAD_REQUEST, "NumberOfBytes must be 1-1024");
if num_bytes < state.config.kms_generate_data_key_min_bytes
|| num_bytes > state.config.kms_generate_data_key_max_bytes
{
return json_err(
StatusCode::BAD_REQUEST,
&format!(
"NumberOfBytes must be {}-{}",
state.config.kms_generate_data_key_min_bytes,
state.config.kms_generate_data_key_max_bytes
),
);
}
let mut bytes = vec![0u8; num_bytes];

View File

@@ -47,9 +47,68 @@ fn s3_error_response(err: S3Error) -> Response {
}
fn storage_err_response(err: myfsio_storage::error::StorageError) -> Response {
if let myfsio_storage::error::StorageError::Io(io_err) = &err {
if let Some(message) = crate::middleware::sha_body::sha256_mismatch_message(io_err) {
return bad_digest_response(message);
}
if let Some(response) = io_error_to_s3_response(io_err) {
return response;
}
}
if let myfsio_storage::error::StorageError::DeleteMarker {
bucket,
key,
version_id,
} = &err
{
let s3_err = S3Error::from_code(S3ErrorCode::NoSuchKey)
.with_resource(format!("/{}/{}", bucket, key))
.with_request_id(uuid::Uuid::new_v4().simple().to_string());
let status = StatusCode::from_u16(s3_err.http_status())
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
let mut resp_headers = HeaderMap::new();
resp_headers.insert("x-amz-delete-marker", "true".parse().unwrap());
if let Ok(vid) = version_id.parse() {
resp_headers.insert("x-amz-version-id", vid);
}
resp_headers.insert("content-type", "application/xml".parse().unwrap());
return (status, resp_headers, s3_err.to_xml()).into_response();
}
s3_error_response(S3Error::from(err))
}
fn io_error_to_s3_response(err: &std::io::Error) -> Option<Response> {
use std::io::ErrorKind;
let message = err.to_string();
let lower = message.to_ascii_lowercase();
let hit_collision = matches!(
err.kind(),
ErrorKind::NotADirectory
| ErrorKind::IsADirectory
| ErrorKind::AlreadyExists
| ErrorKind::DirectoryNotEmpty
) || lower.contains("not a directory")
|| lower.contains("is a directory")
|| lower.contains("file exists")
|| lower.contains("directory not empty");
let hit_name_too_long = matches!(err.kind(), ErrorKind::InvalidFilename)
|| lower.contains("file name too long");
if !hit_collision && !hit_name_too_long {
return None;
}
let code = if hit_name_too_long {
S3ErrorCode::InvalidKey
} else {
S3ErrorCode::InvalidRequest
};
let detail = if hit_name_too_long {
"Object key exceeds the filesystem's per-segment length limit"
} else {
"Object key collides with an existing object path on the storage backend"
};
Some(s3_error_response(S3Error::new(code, detail)))
}
fn trigger_replication(state: &AppState, bucket: &str, key: &str, action: &str) {
let manager = state.replication.clone();
let bucket = bucket.to_string();
@@ -89,6 +148,7 @@ async fn ensure_object_lock_allows_write(
Ok(())
}
Err(myfsio_storage::error::StorageError::ObjectNotFound { .. }) => Ok(()),
Err(myfsio_storage::error::StorageError::DeleteMarker { .. }) => Ok(()),
Err(err) => Err(storage_err_response(err)),
}
}
@@ -237,6 +297,8 @@ pub struct BucketQuery {
pub continuation_token: Option<String>,
#[serde(rename = "start-after")]
pub start_after: Option<String>,
#[serde(rename = "encoding-type")]
pub encoding_type: Option<String>,
pub uploads: Option<String>,
pub delete: Option<String>,
pub versioning: Option<String>,
@@ -391,6 +453,75 @@ pub async fn get_bucket(
Some(marker.clone())
};
if max_keys == 0 {
let has_any = if delimiter.is_empty() {
state
.storage
.list_objects(
&bucket,
&myfsio_common::types::ListParams {
max_keys: 1,
continuation_token: effective_start.clone(),
prefix: if prefix.is_empty() {
None
} else {
Some(prefix.clone())
},
start_after: if is_v2 {
query.start_after.clone()
} else {
None
},
},
)
.await
.map(|r| !r.objects.is_empty())
.unwrap_or(false)
} else {
state
.storage
.list_objects_shallow(
&bucket,
&myfsio_common::types::ShallowListParams {
prefix: prefix.clone(),
delimiter: delimiter.clone(),
max_keys: 1,
continuation_token: effective_start.clone(),
},
)
.await
.map(|r| !r.objects.is_empty() || !r.common_prefixes.is_empty())
.unwrap_or(false)
};
let xml = if is_v2 {
myfsio_xml::response::list_objects_v2_xml(
&bucket,
&prefix,
&delimiter,
0,
&[],
&[],
has_any,
query.continuation_token.as_deref(),
None,
0,
)
} else {
myfsio_xml::response::list_objects_v1_xml(
&bucket,
&prefix,
&marker,
&delimiter,
0,
&[],
&[],
has_any,
None,
)
};
return (StatusCode::OK, [("content-type", "application/xml")], xml).into_response();
}
if delimiter.is_empty() {
let params = myfsio_common::types::ListParams {
max_keys,
@@ -408,15 +539,20 @@ pub async fn get_bucket(
};
match state.storage.list_objects(&bucket, &params).await {
Ok(result) => {
let next_marker = result
let next_marker = if result.is_truncated {
result
.next_continuation_token
.clone()
.or_else(|| result.objects.last().map(|o| o.key.clone()));
.or_else(|| result.objects.last().map(|o| o.key.clone()))
} else {
None
};
let encoding_type = query.encoding_type.as_deref();
let xml = if is_v2 {
let next_token = next_marker
.as_deref()
.map(|s| URL_SAFE.encode(s.as_bytes()));
myfsio_xml::response::list_objects_v2_xml(
myfsio_xml::response::list_objects_v2_xml_with_encoding(
&bucket,
&prefix,
&delimiter,
@@ -427,9 +563,10 @@ pub async fn get_bucket(
query.continuation_token.as_deref(),
next_token.as_deref(),
result.objects.len(),
encoding_type,
)
} else {
myfsio_xml::response::list_objects_v1_xml(
myfsio_xml::response::list_objects_v1_xml_with_encoding(
&bucket,
&prefix,
&marker,
@@ -439,6 +576,7 @@ pub async fn get_bucket(
&[],
result.is_truncated,
next_marker.as_deref(),
encoding_type,
)
};
(StatusCode::OK, [("content-type", "application/xml")], xml).into_response()
@@ -454,12 +592,13 @@ pub async fn get_bucket(
};
match state.storage.list_objects_shallow(&bucket, &params).await {
Ok(result) => {
let encoding_type = query.encoding_type.as_deref();
let xml = if is_v2 {
let next_token = result
.next_continuation_token
.as_deref()
.map(|s| URL_SAFE.encode(s.as_bytes()));
myfsio_xml::response::list_objects_v2_xml(
myfsio_xml::response::list_objects_v2_xml_with_encoding(
&bucket,
&params.prefix,
&delimiter,
@@ -470,9 +609,10 @@ pub async fn get_bucket(
query.continuation_token.as_deref(),
next_token.as_deref(),
result.objects.len() + result.common_prefixes.len(),
encoding_type,
)
} else {
myfsio_xml::response::list_objects_v1_xml(
myfsio_xml::response::list_objects_v1_xml_with_encoding(
&bucket,
&params.prefix,
&marker,
@@ -482,6 +622,7 @@ pub async fn get_bucket(
&result.common_prefixes,
result.is_truncated,
result.next_continuation_token.as_deref(),
encoding_type,
)
};
(StatusCode::OK, [("content-type", "application/xml")], xml).into_response()
@@ -877,6 +1018,47 @@ fn has_upload_checksum(headers: &HeaderMap) -> bool {
|| headers.contains_key("x-amz-checksum-crc32")
}
fn persist_additional_checksums(headers: &HeaderMap, metadata: &mut HashMap<String, String>) {
for algo in [
"sha256", "sha1", "crc32", "crc32c", "crc64nvme",
] {
let header_name = format!("x-amz-checksum-{}", algo);
if let Some(value) = headers.get(&header_name).and_then(|v| v.to_str().ok()) {
let trimmed = value.trim();
if !trimmed.is_empty() {
metadata.insert(format!("__checksum_{}__", algo), trimmed.to_string());
}
}
}
if let Some(value) = headers
.get("x-amz-sdk-checksum-algorithm")
.and_then(|v| v.to_str().ok())
{
let trimmed = value.trim().to_ascii_uppercase();
if !trimmed.is_empty() {
metadata.insert("__checksum_algorithm__".to_string(), trimmed);
}
}
}
fn apply_stored_checksum_headers(resp_headers: &mut HeaderMap, metadata: &HashMap<String, String>) {
for algo in [
"sha256", "sha1", "crc32", "crc32c", "crc64nvme",
] {
if let Some(value) = metadata.get(&format!("__checksum_{}__", algo)) {
if let Ok(parsed) = value.parse() {
resp_headers.insert(
axum::http::HeaderName::from_bytes(
format!("x-amz-checksum-{}", algo).as_bytes(),
)
.unwrap(),
parsed,
);
}
}
}
}
fn validate_upload_checksums(headers: &HeaderMap, data: &[u8]) -> Result<(), Response> {
if let Some(expected) = base64_header_bytes(headers, "content-md5")? {
if expected.len() != 16 || Md5::digest(data).as_slice() != expected.as_slice() {
@@ -906,7 +1088,7 @@ fn validate_upload_checksums(headers: &HeaderMap, data: &[u8]) -> Result<(), Res
Ok(())
}
async fn collect_upload_body(body: Body, aws_chunked: bool) -> Result<Vec<u8>, Response> {
async fn collect_upload_body(body: Body, aws_chunked: bool) -> Result<bytes::Bytes, Response> {
if aws_chunked {
let mut reader = chunked::decode_body(body);
let mut data = Vec::new();
@@ -916,17 +1098,21 @@ async fn collect_upload_body(body: Body, aws_chunked: bool) -> Result<Vec<u8>, R
"Failed to read aws-chunked request body",
))
})?;
return Ok(data);
return Ok(bytes::Bytes::from(data));
}
http_body_util::BodyExt::collect(body)
.await
.map(|collected| collected.to_bytes().to_vec())
.map_err(|_| {
.map(|collected| collected.to_bytes())
.map_err(|err| {
if let Some(message) = crate::middleware::sha_body::sha256_mismatch_message(&err) {
bad_digest_response(message)
} else {
s3_error_response(S3Error::new(
S3ErrorCode::InvalidRequest,
"Failed to read request body",
))
}
})
}
@@ -1122,6 +1308,16 @@ pub async fn put_object(
Ok(tags) => tags,
Err(response) => return response,
};
if let Some(ref tags) = tags {
if tags.len() > state.config.object_tag_limit {
return s3_error_response(S3Error::new(
S3ErrorCode::InvalidTag,
format!("Maximum {} tags allowed", state.config.object_tag_limit),
));
}
}
persist_additional_checksums(&headers, &mut metadata);
let aws_chunked = is_aws_chunked(&headers);
let boxed: myfsio_storage::traits::AsyncReadStream = if has_upload_checksum(&headers) {
@@ -1137,8 +1333,7 @@ pub async fn put_object(
Box::pin(chunked::decode_body(body))
} else {
let stream = tokio_util::io::StreamReader::new(
http_body_util::BodyStream::new(body)
.map_ok(|frame| frame.into_data().unwrap_or_default())
body.into_data_stream()
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)),
);
Box::pin(stream)
@@ -1198,10 +1393,16 @@ pub async fn put_object(
resp_headers
.insert("etag", format!("\"{}\"", etag).parse().unwrap());
}
if let Some(ref vid) = meta.version_id {
if let Ok(value) = vid.parse() {
resp_headers.insert("x-amz-version-id", value);
}
}
resp_headers.insert(
"x-amz-server-side-encryption",
enc_ctx.algorithm.as_str().parse().unwrap(),
);
apply_stored_checksum_headers(&mut resp_headers, &enc_metadata);
notifications::emit_object_created(
&state,
&bucket,
@@ -1231,6 +1432,17 @@ pub async fn put_object(
if let Some(ref etag) = meta.etag {
resp_headers.insert("etag", format!("\"{}\"", etag).parse().unwrap());
}
if let Some(ref vid) = meta.version_id {
if let Ok(value) = vid.parse() {
resp_headers.insert("x-amz-version-id", value);
}
}
let stored = state
.storage
.get_object_metadata(&bucket, &key)
.await
.unwrap_or_default();
apply_stored_checksum_headers(&mut resp_headers, &stored);
notifications::emit_object_created(
&state,
&bucket,
@@ -1360,7 +1572,7 @@ pub async fn get_object(
}
};
let file_size = file.metadata().await.map(|m| m.len()).unwrap_or(0);
let stream = ReaderStream::new(file);
let stream = ReaderStream::with_capacity(file, 256 * 1024);
let body = Body::from_stream(stream);
let meta = head_meta.clone();
@@ -1391,10 +1603,15 @@ pub async fn get_object(
enc_info.algorithm.parse().unwrap(),
);
apply_stored_response_headers(&mut resp_headers, &all_meta);
apply_stored_checksum_headers(&mut resp_headers, &all_meta);
if let Some(ref requested_version) = query.version_id {
if let Ok(value) = requested_version.parse() {
resp_headers.insert("x-amz-version-id", value);
}
} else if let Some(vid) = all_meta.get("__version_id__") {
if let Ok(value) = vid.parse() {
resp_headers.insert("x-amz-version-id", value);
}
}
apply_user_metadata(&mut resp_headers, &meta.metadata);
@@ -1416,7 +1633,7 @@ pub async fn get_object(
match object_result {
Ok((meta, reader)) => {
let stream = ReaderStream::new(reader);
let stream = ReaderStream::with_capacity(reader, 256 * 1024);
let body = Body::from_stream(stream);
let mut headers = HeaderMap::new();
@@ -1435,10 +1652,15 @@ pub async fn get_object(
);
headers.insert("accept-ranges", "bytes".parse().unwrap());
apply_stored_response_headers(&mut headers, &all_meta);
apply_stored_checksum_headers(&mut headers, &all_meta);
if let Some(ref requested_version) = query.version_id {
if let Ok(value) = requested_version.parse() {
headers.insert("x-amz-version-id", value);
}
} else if let Some(ref vid) = meta.version_id {
if let Ok(value) = vid.parse() {
headers.insert("x-amz-version-id", value);
}
}
apply_user_metadata(&mut headers, &meta.metadata);
@@ -1506,11 +1728,16 @@ pub async fn delete_object(
.delete_object_version(&bucket, &key, version_id)
.await
{
Ok(()) => {
Ok(outcome) => {
let mut resp_headers = HeaderMap::new();
if let Ok(value) = version_id.parse() {
if let Some(ref vid) = outcome.version_id {
if let Ok(value) = vid.parse() {
resp_headers.insert("x-amz-version-id", value);
}
}
if outcome.is_delete_marker {
resp_headers.insert("x-amz-delete-marker", "true".parse().unwrap());
}
notifications::emit_object_removed(&state, &bucket, &key, "", "", "", "Delete");
trigger_replication(&state, &bucket, &key, "delete");
(StatusCode::NO_CONTENT, resp_headers).into_response()
@@ -1526,10 +1753,19 @@ pub async fn delete_object(
}
match state.storage.delete_object(&bucket, &key).await {
Ok(()) => {
Ok(outcome) => {
let mut resp_headers = HeaderMap::new();
if let Some(ref vid) = outcome.version_id {
if let Ok(value) = vid.parse() {
resp_headers.insert("x-amz-version-id", value);
}
}
if outcome.is_delete_marker {
resp_headers.insert("x-amz-delete-marker", "true".parse().unwrap());
}
notifications::emit_object_removed(&state, &bucket, &key, "", "", "", "Delete");
trigger_replication(&state, &bucket, &key, "delete");
StatusCode::NO_CONTENT.into_response()
(StatusCode::NO_CONTENT, resp_headers).into_response()
}
Err(e) => storage_err_response(e),
}
@@ -1588,10 +1824,15 @@ pub async fn head_object(
);
headers.insert("accept-ranges", "bytes".parse().unwrap());
apply_stored_response_headers(&mut headers, &all_meta);
apply_stored_checksum_headers(&mut headers, &all_meta);
if let Some(ref requested_version) = query.version_id {
if let Ok(value) = requested_version.parse() {
headers.insert("x-amz-version-id", value);
}
} else if let Some(ref vid) = meta.version_id {
if let Ok(value) = vid.parse() {
headers.insert("x-amz-version-id", value);
}
}
apply_user_metadata(&mut headers, &meta.metadata);
@@ -1624,8 +1865,7 @@ async fn upload_part_handler_with_chunking(
Box::pin(chunked::decode_body(body))
} else {
let stream = tokio_util::io::StreamReader::new(
http_body_util::BodyStream::new(body)
.map_ok(|frame| frame.into_data().unwrap_or_default())
body.into_data_stream()
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)),
);
Box::pin(stream)
@@ -1756,6 +1996,70 @@ async fn complete_multipart_handler(
}
};
if parsed.parts.is_empty() {
return s3_error_response(S3Error::new(
S3ErrorCode::MalformedXML,
"CompleteMultipartUpload requires at least one part",
));
}
let mut last_part_num: u32 = 0;
for p in &parsed.parts {
if p.part_number == 0 {
return s3_error_response(S3Error::new(
S3ErrorCode::InvalidPartOrder,
"Part numbers must be greater than zero",
));
}
if p.part_number <= last_part_num {
return s3_error_response(S3Error::new(
S3ErrorCode::InvalidPartOrder,
"Parts must be specified in ascending order with no duplicates",
));
}
last_part_num = p.part_number;
}
let stored_parts = match state.storage.list_parts(bucket, upload_id).await {
Ok(list) => list,
Err(e) => return storage_err_response(e),
};
let stored_map: HashMap<u32, (String, u64)> = stored_parts
.iter()
.map(|p| (p.part_number, (p.etag.clone(), p.size)))
.collect();
let min_part_size: u64 = state.config.multipart_min_part_size;
let total_parts = parsed.parts.len();
for (idx, p) in parsed.parts.iter().enumerate() {
let stored = match stored_map.get(&p.part_number) {
Some(s) => s,
None => {
return s3_error_response(S3Error::new(
S3ErrorCode::InvalidPart,
format!("Part {} not found", p.part_number),
));
}
};
let client_etag = p.etag.trim().trim_matches('"').to_ascii_lowercase();
let stored_etag = stored.0.trim().trim_matches('"').to_ascii_lowercase();
if !client_etag.is_empty() && client_etag != stored_etag {
return s3_error_response(S3Error::new(
S3ErrorCode::InvalidPart,
format!("ETag mismatch for part {}", p.part_number),
));
}
let is_final = idx + 1 == total_parts;
if !is_final && stored.1 < min_part_size {
return s3_error_response(S3Error::new(
S3ErrorCode::EntityTooSmall,
format!(
"Part {} is smaller than the minimum allowed size of {} bytes",
p.part_number, min_part_size
),
));
}
}
let parts: Vec<PartInfo> = parsed
.parts
.iter()
@@ -1844,7 +2148,8 @@ async fn object_attributes_handler(
if all || attrs.contains("etag") {
if let Some(etag) = &meta.etag {
xml.push_str(&format!("<ETag>{}</ETag>", xml_escape(etag)));
let trimmed = etag.trim_matches('"');
xml.push_str(&format!("<ETag>\"{}\"</ETag>", xml_escape(trimmed)));
}
}
if all || attrs.contains("storageclass") {
@@ -1901,48 +2206,151 @@ async fn copy_object_handler(
return resp;
}
let copy_result = if let Some(version_id) = src_version_id
.as_deref()
.filter(|value| !is_null_version(Some(*value)))
{
let (_meta, mut reader) = match state
.storage
.get_object_version(&src_bucket, &src_key, version_id)
.await
{
Ok(result) => result,
Err(e) => return storage_err_response(e),
};
let mut data = Vec::new();
if let Err(e) = reader.read_to_end(&mut data).await {
return storage_err_response(myfsio_storage::error::StorageError::Io(e));
let metadata_directive = headers
.get("x-amz-metadata-directive")
.and_then(|v| v.to_str().ok())
.map(|v| v.trim().to_ascii_uppercase())
.unwrap_or_else(|| "COPY".to_string());
let tagging_directive = headers
.get("x-amz-tagging-directive")
.and_then(|v| v.to_str().ok())
.map(|v| v.trim().to_ascii_uppercase())
.unwrap_or_else(|| "COPY".to_string());
let replace_metadata = metadata_directive == "REPLACE";
let replace_tagging = tagging_directive == "REPLACE";
let same_object = src_bucket == dst_bucket
&& src_key == dst_key
&& src_version_id.as_deref().unwrap_or("") == "";
if same_object && !replace_metadata && !replace_tagging {
return s3_error_response(S3Error::new(
S3ErrorCode::InvalidRequest,
"This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.",
));
}
let metadata = match state
let source_metadata_existing = match src_version_id.as_deref() {
Some(version_id) if version_id != "null" => {
match state
.storage
.get_object_version_metadata(&src_bucket, &src_key, version_id)
.await
{
Ok(metadata) => metadata,
Err(e) => return storage_err_response(e),
}
}
_ => match state
.storage
.get_object_metadata(&src_bucket, &src_key)
.await
{
Ok(m) => m,
Err(e) => return storage_err_response(e),
},
};
state
let dst_metadata = if replace_metadata {
let mut m: HashMap<String, String> = HashMap::new();
for (request_header, metadata_key, _) in internal_header_pairs() {
if let Some(value) = headers.get(*request_header).and_then(|v| v.to_str().ok()) {
if *request_header == "content-encoding" {
if let Some(decoded_encoding) = decoded_content_encoding(value) {
m.insert((*metadata_key).to_string(), decoded_encoding);
}
} else {
m.insert((*metadata_key).to_string(), value.to_string());
}
}
}
let content_type = guessed_content_type(
dst_key,
headers.get("content-type").and_then(|v| v.to_str().ok()),
);
m.insert("__content_type__".to_string(), content_type);
for (name, value) in headers.iter() {
let name_str = name.as_str();
if let Some(meta_key) = name_str.strip_prefix("x-amz-meta-") {
if let Ok(val) = value.to_str() {
m.insert(meta_key.to_string(), val.to_string());
}
}
}
if let Some(value) = headers
.get("x-amz-storage-class")
.and_then(|v| v.to_str().ok())
{
m.insert("__storage_class__".to_string(), value.to_ascii_uppercase());
}
m
} else {
source_metadata_existing.clone()
};
let (_meta, mut reader) = match src_version_id.as_deref() {
Some(version_id) if version_id != "null" => {
match state
.storage
.get_object_version(&src_bucket, &src_key, version_id)
.await
{
Ok(result) => result,
Err(e) => return storage_err_response(e),
}
}
_ => match state.storage.get_object(&src_bucket, &src_key).await {
Ok(result) => result,
Err(e) => return storage_err_response(e),
},
};
let mut data = Vec::new();
if let Err(e) = reader.read_to_end(&mut data).await {
return storage_err_response(myfsio_storage::error::StorageError::Io(e));
}
let copy_result = state
.storage
.put_object(
dst_bucket,
dst_key,
Box::pin(std::io::Cursor::new(data)),
Some(metadata),
Some(dst_metadata),
)
.await
} else {
state
.storage
.copy_object(&src_bucket, &src_key, dst_bucket, dst_key)
.await
};
.await;
match copy_result {
Ok(meta) => {
if replace_tagging {
let tags = match headers
.get("x-amz-tagging")
.and_then(|value| value.to_str().ok())
.map(parse_tagging_header)
.transpose()
{
Ok(tags) => tags,
Err(response) => return response,
};
if let Some(ref tags) = tags {
if tags.len() > state.config.object_tag_limit {
return s3_error_response(S3Error::new(
S3ErrorCode::InvalidTag,
format!("Maximum {} tags allowed", state.config.object_tag_limit),
));
}
if let Err(e) = state
.storage
.set_object_tags(dst_bucket, dst_key, tags)
.await
{
return storage_err_response(e);
}
} else {
let _ = state
.storage
.set_object_tags(dst_bucket, dst_key, &[])
.await;
}
}
let etag = meta.etag.as_deref().unwrap_or("");
let last_modified = myfsio_xml::response::format_s3_datetime(&meta.last_modified);
let xml = myfsio_xml::response::copy_object_result_xml(etag, &last_modified);
@@ -1975,61 +2383,117 @@ async fn delete_objects_handler(state: &AppState, bucket: &str, body: Body) -> R
}
};
let mut deleted = Vec::new();
let mut errors = Vec::new();
if parsed.objects.len() > 1000 {
return s3_error_response(S3Error::new(
S3ErrorCode::MalformedXML,
"The request must not contain more than 1000 keys",
));
}
for obj in &parsed.objects {
if let Err(message) = match obj.version_id.as_deref() {
use futures::stream::{self, StreamExt};
let results: Vec<(String, Option<String>, Result<myfsio_common::types::DeleteOutcome, (String, String)>)> =
stream::iter(parsed.objects.iter().cloned())
.map(|obj| {
let state = state.clone();
let bucket = bucket.to_string();
async move {
let key = obj.key.clone();
let requested_vid = obj.version_id.clone();
let lock_check: Result<(), (String, String)> = match obj.version_id.as_deref() {
Some(version_id) if version_id != "null" => match state
.storage
.get_object_version_metadata(bucket, &obj.key, version_id)
.get_object_version_metadata(&bucket, &obj.key, version_id)
.await
{
Ok(metadata) => object_lock::can_delete_object(&metadata, false),
Err(err) => Err(S3Error::from(err).message),
},
_ => match state.storage.head_object(bucket, &obj.key).await {
Ok(_) => match state.storage.get_object_metadata(bucket, &obj.key).await {
Ok(metadata) => object_lock::can_delete_object(&metadata, false),
Err(err) => Err(S3Error::from(err).message),
},
Err(myfsio_storage::error::StorageError::ObjectNotFound { .. }) => Ok(()),
Err(err) => Err(S3Error::from(err).message),
},
} {
errors.push((
obj.key.clone(),
S3ErrorCode::AccessDenied.as_str().to_string(),
message,
));
continue;
Ok(metadata) => object_lock::can_delete_object(&metadata, false)
.map_err(|m| {
(S3ErrorCode::AccessDenied.as_str().to_string(), m)
}),
Err(err) => {
let s3err = S3Error::from(err);
Err((s3err.code.as_str().to_string(), s3err.message))
}
let delete_result = if let Some(version_id) = obj.version_id.as_deref() {
if version_id == "null" {
state.storage.delete_object(bucket, &obj.key).await
} else {
state
},
_ => match state.storage.head_object(&bucket, &obj.key).await {
Ok(_) => {
match state
.storage
.delete_object_version(bucket, &obj.key, version_id)
.get_object_metadata(&bucket, &obj.key)
.await
{
Ok(metadata) => object_lock::can_delete_object(&metadata, false)
.map_err(|m| {
(
S3ErrorCode::AccessDenied.as_str().to_string(),
m,
)
}),
Err(err) => {
let s3err = S3Error::from(err);
Err((s3err.code.as_str().to_string(), s3err.message))
}
} else {
state.storage.delete_object(bucket, &obj.key).await
}
}
Err(myfsio_storage::error::StorageError::ObjectNotFound { .. }) => {
Ok(())
}
Err(myfsio_storage::error::StorageError::DeleteMarker { .. }) => {
Ok(())
}
Err(err) => {
let s3err = S3Error::from(err);
Err((s3err.code.as_str().to_string(), s3err.message))
}
},
};
match delete_result {
let result = match lock_check {
Err(e) => Err(e),
Ok(()) => {
notifications::emit_object_removed(state, bucket, &obj.key, "", "", "", "Delete");
trigger_replication(state, bucket, &obj.key, "delete");
deleted.push((obj.key.clone(), obj.version_id.clone()))
let outcome = match obj.version_id.as_deref() {
Some(version_id) if version_id != "null" => {
state
.storage
.delete_object_version(&bucket, &obj.key, version_id)
.await
}
Err(e) => {
_ => state.storage.delete_object(&bucket, &obj.key).await,
};
outcome.map_err(|e| {
let s3err = S3Error::from(e);
errors.push((
obj.key.clone(),
s3err.code.as_str().to_string(),
s3err.message,
));
(s3err.code.as_str().to_string(), s3err.message)
})
}
};
(key, requested_vid, result)
}
})
.buffer_unordered(32)
.collect()
.await;
let mut deleted: Vec<myfsio_xml::response::DeletedEntry> = Vec::new();
let mut errors: Vec<(String, String, String)> = Vec::new();
for (key, requested_vid, result) in results {
match result {
Ok(outcome) => {
notifications::emit_object_removed(state, bucket, &key, "", "", "", "Delete");
trigger_replication(state, bucket, &key, "delete");
let delete_marker_version_id = if outcome.is_delete_marker {
outcome.version_id.clone()
} else {
None
};
deleted.push(myfsio_xml::response::DeletedEntry {
key,
version_id: requested_vid,
delete_marker: outcome.is_delete_marker,
delete_marker_version_id,
});
}
Err((code, message)) => {
errors.push((key, code, message));
}
}
}
@@ -2101,7 +2565,7 @@ async fn range_get_handler(
let length = end - start + 1;
let limited = file.take(length);
let stream = ReaderStream::new(limited);
let stream = ReaderStream::with_capacity(limited, 256 * 1024);
let body = Body::from_stream(stream);
let mut headers = HeaderMap::new();
@@ -2203,7 +2667,8 @@ async fn evaluate_put_preconditions(
}
None
}
Err(myfsio_storage::error::StorageError::ObjectNotFound { .. }) => {
Err(myfsio_storage::error::StorageError::ObjectNotFound { .. })
| Err(myfsio_storage::error::StorageError::DeleteMarker { .. }) => {
if has_if_match {
Some(s3_error_response(S3Error::from_code(
S3ErrorCode::PreconditionFailed,
@@ -2272,9 +2737,20 @@ fn evaluate_copy_preconditions(
}
fn parse_http_date(value: &str) -> Option<DateTime<Utc>> {
DateTime::parse_from_rfc2822(value)
.ok()
.map(|dt| dt.with_timezone(&Utc))
let trimmed = value.trim();
if let Ok(dt) = DateTime::parse_from_rfc2822(trimmed) {
return Some(dt.with_timezone(&Utc));
}
if let Ok(dt) = DateTime::parse_from_rfc3339(trimmed) {
return Some(dt.with_timezone(&Utc));
}
if let Ok(naive) = chrono::NaiveDateTime::parse_from_str(trimmed, "%A, %d-%b-%y %H:%M:%S GMT") {
return Some(naive.and_utc());
}
if let Ok(naive) = chrono::NaiveDateTime::parse_from_str(trimmed, "%a %b %e %H:%M:%S %Y") {
return Some(naive.and_utc());
}
None
}
fn etag_condition_matches(condition: &str, etag: Option<&str>) -> bool {
@@ -2860,6 +3336,7 @@ mod tests {
ui_enabled: false,
templates_dir: manifest_dir.join("templates"),
static_dir: manifest_dir.join("static"),
..ServerConfig::default()
};
(AppState::new(config), tmp)
}

View File

@@ -66,7 +66,7 @@ pub async fn login_submit(
let next = form
.next
.as_deref()
.filter(|n| n.starts_with("/ui/") || *n == "/ui")
.filter(|n| is_allowed_redirect(n, &state.config.allowed_redirect_hosts))
.unwrap_or("/ui/buckets")
.to_string();
Redirect::to(&next).into_response()
@@ -80,6 +80,32 @@ pub async fn login_submit(
}
}
fn is_allowed_redirect(target: &str, allowed_hosts: &[String]) -> bool {
if target == "/ui" || target.starts_with("/ui/") {
return true;
}
let Some(rest) = target
.strip_prefix("https://")
.or_else(|| target.strip_prefix("http://"))
else {
return false;
};
let host = rest
.split('/')
.next()
.unwrap_or_default()
.split('@')
.last()
.unwrap_or_default()
.split(':')
.next()
.unwrap_or_default()
.to_ascii_lowercase();
allowed_hosts
.iter()
.any(|allowed| allowed.eq_ignore_ascii_case(&host))
}
pub async fn logout(Extension(session): Extension<SessionHandle>) -> Response {
session.write(|s| {
s.user_id = None;
@@ -91,16 +117,6 @@ pub async fn logout(Extension(session): Extension<SessionHandle>) -> Response {
Redirect::to("/login").into_response()
}
pub async fn csrf_error_page(
State(state): State<AppState>,
Extension(session): Extension<SessionHandle>,
) -> Response {
let ctx = base_context(&session, None);
let mut resp = render(&state, "csrf_error.html", &ctx);
*resp.status_mut() = StatusCode::FORBIDDEN;
resp
}
pub async fn root_redirect() -> Response {
Redirect::to("/ui/buckets").into_response()
}

View File

@@ -49,6 +49,8 @@ const AWS_QUERY_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC
.remove(b'.')
.remove(b'~');
const UI_OBJECT_BROWSER_MAX_KEYS: usize = 5000;
fn url_templates_for(bucket: &str) -> Value {
json!({
"download": format!("/ui/buckets/{}/objects/KEY_PLACEHOLDER/download", bucket),
@@ -119,6 +121,8 @@ fn storage_status(err: &StorageError) -> StatusCode {
| StorageError::ObjectNotFound { .. }
| StorageError::VersionNotFound { .. }
| StorageError::UploadNotFound(_) => StatusCode::NOT_FOUND,
StorageError::DeleteMarker { .. } => StatusCode::NOT_FOUND,
StorageError::MethodNotAllowed(_) => StatusCode::METHOD_NOT_ALLOWED,
StorageError::InvalidBucketName(_)
| StorageError::InvalidObjectKey(_)
| StorageError::InvalidRange
@@ -185,10 +189,7 @@ fn safe_attachment_filename(key: &str) -> String {
}
fn parse_api_base(state: &AppState) -> String {
std::env::var("API_BASE_URL")
.unwrap_or_else(|_| format!("http://{}", state.config.bind_addr))
.trim_end_matches('/')
.to_string()
state.config.api_base_url.trim_end_matches('/').to_string()
}
fn aws_query_encode(value: &str) -> String {
@@ -905,6 +906,35 @@ pub struct ListObjectsQuery {
pub prefix: Option<String>,
#[serde(default)]
pub start_after: Option<String>,
#[serde(default)]
pub delimiter: Option<String>,
}
fn object_json(bucket_name: &str, o: &myfsio_common::types::ObjectMeta) -> Value {
json!({
"key": o.key,
"size": o.size,
"last_modified": o.last_modified.to_rfc3339(),
"last_modified_iso": o.last_modified.to_rfc3339(),
"last_modified_display": o.last_modified.format("%Y-%m-%d %H:%M:%S").to_string(),
"etag": o.etag.clone().unwrap_or_default(),
"storage_class": o.storage_class.clone().unwrap_or_else(|| "STANDARD".to_string()),
"content_type": o.content_type.clone().unwrap_or_default(),
"download_url": build_ui_object_url(bucket_name, &o.key, "download"),
"preview_url": build_ui_object_url(bucket_name, &o.key, "preview"),
"delete_endpoint": build_ui_object_url(bucket_name, &o.key, "delete"),
"presign_endpoint": build_ui_object_url(bucket_name, &o.key, "presign"),
"metadata_url": build_ui_object_url(bucket_name, &o.key, "metadata"),
"versions_endpoint": build_ui_object_url(bucket_name, &o.key, "versions"),
"restore_template": format!(
"/ui/buckets/{}/objects/{}/restore/VERSION_ID_PLACEHOLDER",
bucket_name,
encode_object_key(&o.key)
),
"tags_url": build_ui_object_url(bucket_name, &o.key, "tags"),
"copy_url": build_ui_object_url(bucket_name, &o.key, "copy"),
"move_url": build_ui_object_url(bucket_name, &o.key, "move"),
})
}
pub async fn list_bucket_objects(
@@ -918,6 +948,49 @@ pub async fn list_bucket_objects(
}
let max_keys = q.max_keys.unwrap_or(1000).min(5000);
let versioning_enabled = state
.storage
.is_versioning_enabled(&bucket_name)
.await
.unwrap_or(false);
let stats = state.storage.bucket_stats(&bucket_name).await.ok();
let total_count = stats.as_ref().map(|s| s.objects).unwrap_or(0);
let use_shallow = q.delimiter.as_deref() == Some("/");
if use_shallow {
let params = myfsio_common::types::ShallowListParams {
prefix: q.prefix.clone().unwrap_or_default(),
delimiter: "/".to_string(),
max_keys,
continuation_token: q.continuation_token.clone(),
};
return match state
.storage
.list_objects_shallow(&bucket_name, &params)
.await
{
Ok(res) => {
let objects: Vec<Value> = res
.objects
.iter()
.map(|o| object_json(&bucket_name, o))
.collect();
Json(json!({
"versioning_enabled": versioning_enabled,
"total_count": total_count,
"is_truncated": res.is_truncated,
"next_continuation_token": res.next_continuation_token,
"url_templates": url_templates_for(&bucket_name),
"objects": objects,
"common_prefixes": res.common_prefixes,
}))
.into_response()
}
Err(e) => storage_json_error(e),
};
}
let params = ListParams {
max_keys,
continuation_token: q.continuation_token.clone(),
@@ -925,46 +998,12 @@ pub async fn list_bucket_objects(
start_after: q.start_after.clone(),
};
let versioning_enabled = state
.storage
.is_versioning_enabled(&bucket_name)
.await
.unwrap_or(false);
let stats = state.storage.bucket_stats(&bucket_name).await.ok();
let total_count = stats.as_ref().map(|s| s.objects).unwrap_or(0);
match state.storage.list_objects(&bucket_name, &params).await {
Ok(res) => {
let objects: Vec<Value> = res
.objects
.iter()
.map(|o| {
json!({
"key": o.key,
"size": o.size,
"last_modified": o.last_modified.to_rfc3339(),
"last_modified_iso": o.last_modified.to_rfc3339(),
"last_modified_display": o.last_modified.format("%Y-%m-%d %H:%M:%S").to_string(),
"etag": o.etag.clone().unwrap_or_default(),
"storage_class": o.storage_class.clone().unwrap_or_else(|| "STANDARD".to_string()),
"content_type": o.content_type.clone().unwrap_or_default(),
"download_url": build_ui_object_url(&bucket_name, &o.key, "download"),
"preview_url": build_ui_object_url(&bucket_name, &o.key, "preview"),
"delete_endpoint": build_ui_object_url(&bucket_name, &o.key, "delete"),
"presign_endpoint": build_ui_object_url(&bucket_name, &o.key, "presign"),
"metadata_url": build_ui_object_url(&bucket_name, &o.key, "metadata"),
"versions_endpoint": build_ui_object_url(&bucket_name, &o.key, "versions"),
"restore_template": format!(
"/ui/buckets/{}/objects/{}/restore/VERSION_ID_PLACEHOLDER",
bucket_name,
encode_object_key(&o.key)
),
"tags_url": build_ui_object_url(&bucket_name, &o.key, "tags"),
"copy_url": build_ui_object_url(&bucket_name, &o.key, "copy"),
"move_url": build_ui_object_url(&bucket_name, &o.key, "move"),
})
})
.map(|o| object_json(&bucket_name, o))
.collect();
Json(json!({
@@ -1007,39 +1046,62 @@ pub async fn stream_bucket_objects(
let stats = state.storage.bucket_stats(&bucket_name).await.ok();
let total_count = stats.as_ref().map(|s| s.objects).unwrap_or(0);
let mut lines: Vec<String> = Vec::new();
lines.push(
json!({
let use_delimiter = q.delimiter.as_deref() == Some("/");
let prefix = q.prefix.clone().unwrap_or_default();
let (tx, rx) = tokio::sync::mpsc::channel::<Result<bytes::Bytes, std::io::Error>>(64);
let meta_line = json!({
"type": "meta",
"url_templates": url_templates_for(&bucket_name),
"versioning_enabled": versioning_enabled,
})
.to_string(),
);
lines.push(json!({ "type": "count", "total_count": total_count }).to_string());
.to_string()
+ "\n";
let count_line = json!({ "type": "count", "total_count": total_count }).to_string() + "\n";
let use_delimiter = q.delimiter.as_deref() == Some("/");
let prefix = q.prefix.clone().unwrap_or_default();
let storage = state.storage.clone();
let bucket = bucket_name.clone();
tokio::spawn(async move {
if tx
.send(Ok(bytes::Bytes::from(meta_line.into_bytes())))
.await
.is_err()
{
return;
}
if tx
.send(Ok(bytes::Bytes::from(count_line.into_bytes())))
.await
.is_err()
{
return;
}
if use_delimiter {
let mut token: Option<String> = None;
loop {
let params = myfsio_common::types::ShallowListParams {
prefix: prefix.clone(),
delimiter: "/".to_string(),
max_keys: 5000,
continuation_token: None,
max_keys: UI_OBJECT_BROWSER_MAX_KEYS,
continuation_token: token.clone(),
};
match state
.storage
.list_objects_shallow(&bucket_name, &params)
.await
{
match storage.list_objects_shallow(&bucket, &params).await {
Ok(res) => {
for p in &res.common_prefixes {
lines.push(json!({ "type": "folder", "prefix": p }).to_string());
let line = json!({ "type": "folder", "prefix": p }).to_string() + "\n";
if tx
.send(Ok(bytes::Bytes::from(line.into_bytes())))
.await
.is_err()
{
return;
}
}
for o in &res.objects {
lines.push(
json!({
let line = json!({
"type": "object",
"key": o.key,
"size": o.size,
@@ -1049,11 +1111,28 @@ pub async fn stream_bucket_objects(
"etag": o.etag.clone().unwrap_or_default(),
"storage_class": o.storage_class.clone().unwrap_or_else(|| "STANDARD".to_string()),
})
.to_string(),
);
.to_string()
+ "\n";
if tx
.send(Ok(bytes::Bytes::from(line.into_bytes())))
.await
.is_err()
{
return;
}
}
if !res.is_truncated || res.next_continuation_token.is_none() {
break;
}
token = res.next_continuation_token;
}
Err(e) => {
let line =
json!({ "type": "error", "error": e.to_string() }).to_string() + "\n";
let _ = tx.send(Ok(bytes::Bytes::from(line.into_bytes()))).await;
return;
}
}
Err(e) => lines.push(json!({ "type": "error", "error": e.to_string() }).to_string()),
}
} else {
let mut token: Option<String> = None;
@@ -1068,11 +1147,10 @@ pub async fn stream_bucket_objects(
},
start_after: None,
};
match state.storage.list_objects(&bucket_name, &params).await {
match storage.list_objects(&bucket, &params).await {
Ok(res) => {
for o in &res.objects {
lines.push(
json!({
let line = json!({
"type": "object",
"key": o.key,
"size": o.size,
@@ -1082,8 +1160,15 @@ pub async fn stream_bucket_objects(
"etag": o.etag.clone().unwrap_or_default(),
"storage_class": o.storage_class.clone().unwrap_or_else(|| "STANDARD".to_string()),
})
.to_string(),
);
.to_string()
+ "\n";
if tx
.send(Ok(bytes::Bytes::from(line.into_bytes())))
.await
.is_err()
{
return;
}
}
if !res.is_truncated || res.next_continuation_token.is_none() {
break;
@@ -1091,21 +1176,32 @@ pub async fn stream_bucket_objects(
token = res.next_continuation_token;
}
Err(e) => {
lines.push(json!({ "type": "error", "error": e.to_string() }).to_string());
break;
let line =
json!({ "type": "error", "error": e.to_string() }).to_string() + "\n";
let _ = tx.send(Ok(bytes::Bytes::from(line.into_bytes()))).await;
return;
}
}
}
}
lines.push(json!({ "type": "done" }).to_string());
let done_line = json!({ "type": "done" }).to_string() + "\n";
let _ = tx
.send(Ok(bytes::Bytes::from(done_line.into_bytes())))
.await;
});
let stream = tokio_stream::wrappers::ReceiverStream::new(rx);
let body = Body::from_stream(stream);
let body = lines.join("\n") + "\n";
let mut headers = HeaderMap::new();
headers.insert(
header::CONTENT_TYPE,
"application/x-ndjson; charset=utf-8".parse().unwrap(),
);
headers.insert(header::CACHE_CONTROL, "no-cache".parse().unwrap());
headers.insert("x-accel-buffering", "no".parse().unwrap());
(StatusCode::OK, headers, body).into_response()
}
@@ -1123,7 +1219,7 @@ pub async fn list_bucket_folders(
let params = myfsio_common::types::ShallowListParams {
prefix: prefix.clone(),
delimiter: "/".to_string(),
max_keys: 5000,
max_keys: UI_OBJECT_BROWSER_MAX_KEYS,
continuation_token: None,
};
match state
@@ -2408,8 +2504,11 @@ async fn update_object_tags(state: &AppState, bucket: &str, key: &str, body: Bod
Err(response) => return response,
};
if payload.tags.len() > 50 {
return json_error(StatusCode::BAD_REQUEST, "Maximum 50 tags allowed");
if payload.tags.len() > state.config.object_tag_limit {
return json_error(
StatusCode::BAD_REQUEST,
format!("Maximum {} tags allowed", state.config.object_tag_limit),
);
}
let tags = payload
@@ -2502,7 +2601,7 @@ async fn move_object_json(state: &AppState, bucket: &str, key: &str, body: Body)
match state.storage.copy_object(bucket, key, dest_bucket, dest_key).await {
Ok(_) => match state.storage.delete_object(bucket, key).await {
Ok(()) => {
Ok(_) => {
super::trigger_replication(state, dest_bucket, dest_key, "write");
super::trigger_replication(state, bucket, key, "delete");
Json(json!({
@@ -2577,7 +2676,7 @@ async fn delete_object_json(
}
match state.storage.delete_object(bucket, key).await {
Ok(()) => {
Ok(_) => {
super::trigger_replication(state, bucket, key, "delete");
Json(json!({
"status": "ok",
@@ -2841,13 +2940,22 @@ pub async fn bulk_delete_objects(
"No objects found under the selected folders",
);
}
if keys.len() > state.config.bulk_delete_max_keys {
return json_error(
StatusCode::BAD_REQUEST,
format!(
"Bulk delete supports at most {} keys",
state.config.bulk_delete_max_keys
),
);
}
let mut deleted = Vec::new();
let mut errors = Vec::new();
for key in keys {
match state.storage.delete_object(&bucket_name, &key).await {
Ok(()) => {
Ok(_) => {
super::trigger_replication(&state, &bucket_name, &key, "delete");
if payload.purge_versions {
if let Err(err) =

View File

@@ -1,8 +1,10 @@
use std::collections::HashMap;
use axum::body::Body;
use axum::extract::{Extension, Form, Path, Query, State};
use axum::http::{header, HeaderMap, StatusCode};
use axum::response::{IntoResponse, Redirect, Response};
use http_body_util::BodyExt;
use serde_json::{json, Value};
use tera::Context;
@@ -203,6 +205,57 @@ fn wants_json(headers: &HeaderMap) -> bool {
.unwrap_or(false)
}
async fn parse_form_any(
headers: &HeaderMap,
body: Body,
) -> Result<HashMap<String, String>, String> {
let content_type = headers
.get(header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
let is_multipart = content_type
.to_ascii_lowercase()
.starts_with("multipart/form-data");
let bytes = body
.collect()
.await
.map_err(|e| format!("Failed to read request body: {}", e))?
.to_bytes();
if is_multipart {
let boundary = multer::parse_boundary(&content_type)
.map_err(|_| "Missing multipart boundary".to_string())?;
let stream = futures::stream::once(async move { Ok::<_, std::io::Error>(bytes) });
let mut multipart = multer::Multipart::new(stream, boundary);
let mut out = HashMap::new();
while let Some(field) = multipart
.next_field()
.await
.map_err(|e| format!("Malformed multipart body: {}", e))?
{
let name = match field.name() {
Some(name) => name.to_string(),
None => continue,
};
if field.file_name().is_some() {
continue;
}
let value = field
.text()
.await
.map_err(|e| format!("Invalid multipart field '{}': {}", name, e))?;
out.insert(name, value);
}
Ok(out)
} else {
let parsed: Vec<(String, String)> = serde_urlencoded::from_bytes(&bytes)
.map_err(|e| format!("Invalid form body: {}", e))?;
Ok(parsed.into_iter().collect())
}
}
fn bucket_tab_redirect(bucket_name: &str, tab: &str) -> Response {
Redirect::to(&format!("/ui/buckets/{}?tab={}", bucket_name, tab)).into_response()
}
@@ -231,10 +284,7 @@ fn default_public_policy(bucket_name: &str) -> String {
}
fn parse_api_base(state: &AppState) -> (String, String) {
let api_base = std::env::var("API_BASE_URL")
.unwrap_or_else(|_| format!("http://{}", state.config.bind_addr))
.trim_end_matches('/')
.to_string();
let api_base = state.config.api_base_url.trim_end_matches('/').to_string();
let api_host = api_base
.split("://")
.nth(1)
@@ -1173,16 +1223,13 @@ pub async fn sites_dashboard(
ctx.insert("connections", &conns);
ctx.insert(
"config_site_id",
&std::env::var("SITE_ID").unwrap_or_default(),
&state.config.site_id.clone().unwrap_or_default(),
);
ctx.insert(
"config_site_endpoint",
&std::env::var("SITE_ENDPOINT").unwrap_or_default(),
);
ctx.insert(
"config_site_region",
&std::env::var("SITE_REGION").unwrap_or_else(|_| state.config.region.clone()),
&state.config.site_endpoint.clone().unwrap_or_default(),
);
ctx.insert("config_site_region", &state.config.site_region);
ctx.insert("topology", &json!({"sites": [], "connections": []}));
render(&state, "sites.html", &ctx)
}
@@ -2119,9 +2166,26 @@ pub async fn create_bucket(
State(state): State<AppState>,
Extension(session): Extension<SessionHandle>,
headers: HeaderMap,
axum::extract::Form(form): axum::extract::Form<CreateBucketForm>,
body: Body,
) -> Response {
let wants_json = wants_json(&headers);
let form = match parse_form_any(&headers, body).await {
Ok(fields) => CreateBucketForm {
bucket_name: fields.get("bucket_name").cloned().unwrap_or_default(),
csrf_token: fields.get("csrf_token").cloned().unwrap_or_default(),
},
Err(message) => {
if wants_json {
return (
StatusCode::BAD_REQUEST,
axum::Json(json!({ "error": message })),
)
.into_response();
}
session.write(|s| s.push_flash("danger", message));
return Redirect::to("/ui/buckets").into_response();
}
};
let bucket_name = form.bucket_name.trim().to_string();
if bucket_name.is_empty() {

View File

@@ -304,8 +304,7 @@ pub fn create_ui_router(state: state::AppState) -> Router {
let public = Router::new()
.route("/login", get(ui::login_page).post(ui::login_submit))
.route("/logout", post(ui::logout).get(ui::logout))
.route("/csrf-error", get(ui::csrf_error_page));
.route("/logout", post(ui::logout).get(ui::logout));
let session_state = middleware::SessionLayerState {
store: state.sessions.clone(),
@@ -317,7 +316,10 @@ pub fn create_ui_router(state: state::AppState) -> Router {
protected
.merge(public)
.fallback(ui::not_found_page)
.layer(axum::middleware::from_fn(middleware::csrf_layer))
.layer(axum::middleware::from_fn_with_state(
state.clone(),
middleware::csrf_layer,
))
.layer(axum::middleware::from_fn_with_state(
session_state,
middleware::session_layer,
@@ -333,7 +335,20 @@ pub fn create_ui_router(state: state::AppState) -> Router {
}
pub fn create_router(state: state::AppState) -> Router {
let mut router = Router::new()
let default_rate_limit = middleware::RateLimitLayerState::with_per_op(
state.config.ratelimit_default,
state.config.ratelimit_list_buckets,
state.config.ratelimit_bucket_ops,
state.config.ratelimit_object_ops,
state.config.ratelimit_head_ops,
state.config.num_trusted_proxies,
);
let admin_rate_limit = middleware::RateLimitLayerState::new(
state.config.ratelimit_admin,
state.config.num_trusted_proxies,
);
let mut api_router = Router::new()
.route("/myfsio/health", axum::routing::get(handlers::health_check))
.route("/", axum::routing::get(handlers::list_buckets))
.route(
@@ -362,7 +377,7 @@ pub fn create_router(state: state::AppState) -> Router {
);
if state.config.kms_enabled {
router = router
api_router = api_router
.route(
"/kms/keys",
axum::routing::get(handlers::kms::list_keys).post(handlers::kms::create_key),
@@ -415,7 +430,17 @@ pub fn create_router(state: state::AppState) -> Router {
);
}
router = router
api_router = api_router
.layer(axum::middleware::from_fn_with_state(
state.clone(),
middleware::auth_layer,
))
.layer(axum::middleware::from_fn_with_state(
default_rate_limit,
middleware::rate_limit_layer,
));
let admin_router = Router::new()
.route(
"/admin/site",
axum::routing::get(handlers::admin::get_local_site)
@@ -546,14 +571,87 @@ pub fn create_router(state: state::AppState) -> Router {
.route(
"/admin/integrity/history",
axum::routing::get(handlers::admin::integrity_history),
);
router
)
.layer(axum::middleware::from_fn_with_state(
state.clone(),
middleware::auth_layer,
))
.layer(axum::middleware::from_fn_with_state(
admin_rate_limit,
middleware::rate_limit_layer,
));
let request_body_timeout =
std::time::Duration::from_secs(state.config.request_body_timeout_secs);
api_router
.merge(admin_router)
.layer(axum::middleware::from_fn(middleware::server_header))
.layer(cors_layer(&state.config))
.layer(tower_http::compression::CompressionLayer::new())
.layer(tower_http::timeout::RequestBodyTimeoutLayer::new(
request_body_timeout,
))
.with_state(state)
}
fn cors_layer(config: &config::ServerConfig) -> tower_http::cors::CorsLayer {
use axum::http::{HeaderName, HeaderValue, Method};
use tower_http::cors::{Any, CorsLayer};
let mut layer = CorsLayer::new();
if config.cors_origins.iter().any(|origin| origin == "*") {
layer = layer.allow_origin(Any);
} else {
let origins = config
.cors_origins
.iter()
.filter_map(|origin| HeaderValue::from_str(origin).ok())
.collect::<Vec<_>>();
if !origins.is_empty() {
layer = layer.allow_origin(origins);
}
}
let methods = config
.cors_methods
.iter()
.filter_map(|method| method.parse::<Method>().ok())
.collect::<Vec<_>>();
if !methods.is_empty() {
layer = layer.allow_methods(methods);
}
if config.cors_allow_headers.iter().any(|header| header == "*") {
layer = layer.allow_headers(Any);
} else {
let headers = config
.cors_allow_headers
.iter()
.filter_map(|header| header.parse::<HeaderName>().ok())
.collect::<Vec<_>>();
if !headers.is_empty() {
layer = layer.allow_headers(headers);
}
}
if config
.cors_expose_headers
.iter()
.any(|header| header == "*")
{
layer = layer.expose_headers(Any);
} else {
let headers = config
.cors_expose_headers
.iter()
.filter_map(|header| header.parse::<HeaderName>().ok())
.collect::<Vec<_>>();
if !headers.is_empty() {
layer = layer.expose_headers(headers);
}
}
layer
}

View File

@@ -28,10 +28,19 @@ enum Command {
#[tokio::main]
async fn main() {
load_env_files();
tracing_subscriber::fmt::init();
init_tracing();
let cli = Cli::parse();
let config = ServerConfig::from_env();
if !config
.ratelimit_storage_uri
.eq_ignore_ascii_case("memory://")
{
tracing::warn!(
"RATE_LIMIT_STORAGE_URI={} is not supported yet; using in-memory rate limits",
config.ratelimit_storage_uri
);
}
if cli.reset_cred {
reset_admin_credentials(&config);
@@ -114,7 +123,10 @@ async fn main() {
std::sync::Arc::new(myfsio_server::services::lifecycle::LifecycleService::new(
state.storage.clone(),
config.storage_root.clone(),
myfsio_server::services::lifecycle::LifecycleConfig::default(),
myfsio_server::services::lifecycle::LifecycleConfig {
interval_seconds: 3600,
max_history_per_bucket: config.lifecycle_max_history_per_bucket,
},
));
bg_handles.push(lifecycle.start_background());
tracing::info!("Lifecycle manager background service started");
@@ -177,8 +189,16 @@ async fn main() {
let shutdown = shutdown_signal_shared();
let api_shutdown = shutdown.clone();
let api_listener = axum::serve::ListenerExt::tap_io(api_listener, |stream| {
if let Err(err) = stream.set_nodelay(true) {
tracing::trace!("failed to set TCP_NODELAY on api socket: {}", err);
}
});
let api_task = tokio::spawn(async move {
axum::serve(api_listener, api_app)
axum::serve(
api_listener,
api_app.into_make_service_with_connect_info::<std::net::SocketAddr>(),
)
.with_graceful_shutdown(async move {
api_shutdown.notified().await;
})
@@ -187,6 +207,11 @@ async fn main() {
let ui_task = if let (Some(listener), Some(app)) = (ui_listener, ui_app) {
let ui_shutdown = shutdown.clone();
let listener = axum::serve::ListenerExt::tap_io(listener, |stream| {
if let Err(err) = stream.set_nodelay(true) {
tracing::trace!("failed to set TCP_NODELAY on ui socket: {}", err);
}
});
Some(tokio::spawn(async move {
axum::serve(listener, app)
.with_graceful_shutdown(async move {
@@ -228,15 +253,43 @@ fn print_config_summary(config: &ServerConfig) {
println!("IAM config: {}", config.iam_config_path.display());
println!("Region: {}", config.region);
println!("Encryption enabled: {}", config.encryption_enabled);
println!(
"Encryption chunk size: {} bytes",
config.encryption_chunk_size_bytes
);
println!("KMS enabled: {}", config.kms_enabled);
println!(
"KMS data key bounds: {}-{} bytes",
config.kms_generate_data_key_min_bytes, config.kms_generate_data_key_max_bytes
);
println!("GC enabled: {}", config.gc_enabled);
println!(
"GC interval: {} hours, dry run: {}",
config.gc_interval_hours, config.gc_dry_run
);
println!("Integrity enabled: {}", config.integrity_enabled);
println!("Lifecycle enabled: {}", config.lifecycle_enabled);
println!(
"Lifecycle history limit: {}",
config.lifecycle_max_history_per_bucket
);
println!(
"Website hosting enabled: {}",
config.website_hosting_enabled
);
println!("Site sync enabled: {}", config.site_sync_enabled);
println!("API base URL: {}", config.api_base_url);
println!(
"Object key max: {} bytes, tag limit: {}",
config.object_key_max_length_bytes, config.object_tag_limit
);
println!(
"Rate limits: default {} per {}s, admin {} per {}s",
config.ratelimit_default.max_requests,
config.ratelimit_default.window_seconds,
config.ratelimit_admin.max_requests,
config.ratelimit_admin.window_seconds
);
println!(
"Metrics history enabled: {}",
config.metrics_history_enabled
@@ -256,6 +309,32 @@ fn validate_config(config: &ServerConfig) -> Vec<String> {
if config.presigned_url_min_expiry > config.presigned_url_max_expiry {
issues.push("CRITICAL: PRESIGNED_URL_MIN_EXPIRY_SECONDS cannot exceed PRESIGNED_URL_MAX_EXPIRY_SECONDS.".to_string());
}
if config.encryption_chunk_size_bytes == 0 {
issues.push("CRITICAL: ENCRYPTION_CHUNK_SIZE_BYTES must be greater than zero.".to_string());
}
if config.kms_generate_data_key_min_bytes == 0 {
issues.push(
"CRITICAL: KMS_GENERATE_DATA_KEY_MIN_BYTES must be greater than zero.".to_string(),
);
}
if config.kms_generate_data_key_min_bytes > config.kms_generate_data_key_max_bytes {
issues.push("CRITICAL: KMS_GENERATE_DATA_KEY_MIN_BYTES cannot exceed KMS_GENERATE_DATA_KEY_MAX_BYTES.".to_string());
}
if config.gc_interval_hours <= 0.0 {
issues.push("CRITICAL: GC_INTERVAL_HOURS must be greater than zero.".to_string());
}
if config.bucket_config_cache_ttl_seconds < 0.0 {
issues.push("CRITICAL: BUCKET_CONFIG_CACHE_TTL_SECONDS cannot be negative.".to_string());
}
if !config
.ratelimit_storage_uri
.eq_ignore_ascii_case("memory://")
{
issues.push(format!(
"WARNING: RATE_LIMIT_STORAGE_URI={} is not supported yet; using in-memory limits.",
config.ratelimit_storage_uri
));
}
if let Err(err) = std::fs::create_dir_all(&config.storage_root) {
issues.push(format!(
"CRITICAL: Cannot create storage root {}: {}",
@@ -286,6 +365,17 @@ fn validate_config(config: &ServerConfig) -> Vec<String> {
issues
}
fn init_tracing() {
use tracing_subscriber::EnvFilter;
let filter = EnvFilter::try_from_env("RUST_LOG")
.or_else(|_| {
EnvFilter::try_new(std::env::var("LOG_LEVEL").unwrap_or_else(|_| "INFO".to_string()))
})
.unwrap_or_else(|_| EnvFilter::new("INFO"));
tracing_subscriber::fmt().with_env_filter(filter).init();
}
fn shutdown_signal_shared() -> std::sync::Arc<tokio::sync::Notify> {
std::sync::Arc::new(tokio::sync::Notify::new())
}
@@ -419,8 +509,49 @@ fn reset_admin_credentials(config: &ServerConfig) {
std::process::exit(1);
}
println!("Backed up existing IAM config to {}", backup.display());
prune_iam_backups(&config.iam_config_path, 5);
}
ensure_iam_bootstrap(config);
println!("Admin credentials reset.");
}
fn prune_iam_backups(iam_path: &std::path::Path, keep: usize) {
let parent = match iam_path.parent() {
Some(p) => p,
None => return,
};
let stem = match iam_path.file_stem().and_then(|s| s.to_str()) {
Some(s) => s,
None => return,
};
let prefix = format!("{}.bak-", stem);
let entries = match std::fs::read_dir(parent) {
Ok(entries) => entries,
Err(_) => return,
};
let mut backups: Vec<(i64, std::path::PathBuf)> = entries
.filter_map(|e| e.ok())
.filter_map(|e| {
let path = e.path();
let name = path.file_name()?.to_str()?;
let rest = name.strip_prefix(&prefix)?;
let ts: i64 = rest.parse().ok()?;
Some((ts, path))
})
.collect();
backups.sort_by(|a, b| b.0.cmp(&a.0));
for (_, path) in backups.into_iter().skip(keep) {
if let Err(err) = std::fs::remove_file(&path) {
eprintln!(
"Failed to remove old IAM backup {}: {}",
path.display(),
err
);
} else {
println!("Pruned old IAM backup {}", path.display());
}
}
}

View File

@@ -12,9 +12,36 @@ use serde_json::Value;
use std::time::Instant;
use tokio::io::AsyncReadExt;
use crate::middleware::sha_body::{is_hex_sha256, Sha256VerifyBody};
use crate::services::acl::acl_from_bucket_config;
use crate::state::AppState;
fn wrap_body_for_sha256_verification(req: &mut Request) {
let declared = match req
.headers()
.get("x-amz-content-sha256")
.and_then(|v| v.to_str().ok())
{
Some(v) => v.to_string(),
None => return,
};
if !is_hex_sha256(&declared) {
return;
}
let is_chunked = req
.headers()
.get("content-encoding")
.and_then(|v| v.to_str().ok())
.map(|v| v.to_ascii_lowercase().contains("aws-chunked"))
.unwrap_or(false);
if is_chunked {
return;
}
let body = std::mem::replace(req.body_mut(), axum::body::Body::empty());
let wrapped = Sha256VerifyBody::new(body, declared);
*req.body_mut() = axum::body::Body::new(wrapped);
}
#[derive(Clone, Debug)]
struct OriginalCanonicalPath(String);
@@ -475,6 +502,7 @@ pub async fn auth_layer(State(state): State<AppState>, mut req: Request, next: N
error_response(err, &auth_path)
} else {
req.extensions_mut().insert(principal);
wrap_body_for_sha256_verification(&mut req);
next.run(req).await
}
}
@@ -1102,7 +1130,9 @@ fn verify_sigv4_header(state: &AppState, req: &Request, auth_str: &str) -> AuthR
let parts: Vec<&str> = auth_str
.strip_prefix("AWS4-HMAC-SHA256 ")
.unwrap()
.split(", ")
.split(',')
.map(str::trim)
.filter(|s| !s.is_empty())
.collect();
if parts.len() != 3 {
@@ -1112,9 +1142,24 @@ fn verify_sigv4_header(state: &AppState, req: &Request, auth_str: &str) -> AuthR
));
}
let credential = parts[0].strip_prefix("Credential=").unwrap_or("");
let signed_headers_str = parts[1].strip_prefix("SignedHeaders=").unwrap_or("");
let provided_signature = parts[2].strip_prefix("Signature=").unwrap_or("");
let mut credential: &str = "";
let mut signed_headers_str: &str = "";
let mut provided_signature: &str = "";
for part in &parts {
if let Some(v) = part.strip_prefix("Credential=") {
credential = v;
} else if let Some(v) = part.strip_prefix("SignedHeaders=") {
signed_headers_str = v;
} else if let Some(v) = part.strip_prefix("Signature=") {
provided_signature = v;
}
}
if credential.is_empty() || signed_headers_str.is_empty() || provided_signature.is_empty() {
return AuthResult::Denied(S3Error::new(
S3ErrorCode::InvalidArgument,
"Malformed Authorization header",
));
}
let cred_parts: Vec<&str> = credential.split('/').collect();
if cred_parts.len() != 5 {
@@ -1299,7 +1344,7 @@ fn verify_sigv4_query(state: &AppState, req: &Request) -> AuthResult {
}
if elapsed < -(state.config.sigv4_timestamp_tolerance_secs as i64) {
return AuthResult::Denied(S3Error::new(
S3ErrorCode::AccessDenied,
S3ErrorCode::RequestTimeTooSkewed,
"Request is too far in the future",
));
}
@@ -1369,8 +1414,11 @@ fn check_timestamp_freshness(amz_date: &str, tolerance_secs: u64) -> Option<S3Er
if diff > tolerance_secs {
return Some(S3Error::new(
S3ErrorCode::AccessDenied,
"Request timestamp too old or too far in the future",
S3ErrorCode::RequestTimeTooSkewed,
format!(
"The difference between the request time and the server's time is too large ({}s, tolerance {}s)",
diff, tolerance_secs
),
));
}
None

View File

@@ -1,7 +1,10 @@
mod auth;
pub mod ratelimit;
pub mod session;
pub(crate) mod sha_body;
pub use auth::auth_layer;
pub use ratelimit::{rate_limit_layer, RateLimitLayerState};
pub use session::{csrf_layer, session_layer, SessionHandle, SessionLayerState};
use axum::extract::{Request, State};

View File

@@ -0,0 +1,313 @@
use std::collections::HashMap;
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::{Duration, Instant};
use axum::extract::{ConnectInfo, Request, State};
use axum::http::{header, Method, StatusCode};
use axum::middleware::Next;
use axum::response::{IntoResponse, Response};
use parking_lot::Mutex;
use crate::config::RateLimitSetting;
#[derive(Clone)]
pub struct RateLimitLayerState {
default_limiter: Arc<FixedWindowLimiter>,
list_buckets_limiter: Option<Arc<FixedWindowLimiter>>,
bucket_ops_limiter: Option<Arc<FixedWindowLimiter>>,
object_ops_limiter: Option<Arc<FixedWindowLimiter>>,
head_ops_limiter: Option<Arc<FixedWindowLimiter>>,
num_trusted_proxies: usize,
}
impl RateLimitLayerState {
pub fn new(setting: RateLimitSetting, num_trusted_proxies: usize) -> Self {
Self {
default_limiter: Arc::new(FixedWindowLimiter::new(setting)),
list_buckets_limiter: None,
bucket_ops_limiter: None,
object_ops_limiter: None,
head_ops_limiter: None,
num_trusted_proxies,
}
}
pub fn with_per_op(
default: RateLimitSetting,
list_buckets: RateLimitSetting,
bucket_ops: RateLimitSetting,
object_ops: RateLimitSetting,
head_ops: RateLimitSetting,
num_trusted_proxies: usize,
) -> Self {
Self {
default_limiter: Arc::new(FixedWindowLimiter::new(default)),
list_buckets_limiter: (list_buckets != default)
.then(|| Arc::new(FixedWindowLimiter::new(list_buckets))),
bucket_ops_limiter: (bucket_ops != default)
.then(|| Arc::new(FixedWindowLimiter::new(bucket_ops))),
object_ops_limiter: (object_ops != default)
.then(|| Arc::new(FixedWindowLimiter::new(object_ops))),
head_ops_limiter: (head_ops != default)
.then(|| Arc::new(FixedWindowLimiter::new(head_ops))),
num_trusted_proxies,
}
}
fn select_limiter(&self, req: &Request) -> &Arc<FixedWindowLimiter> {
let path = req.uri().path();
let method = req.method();
if path == "/" && *method == Method::GET {
if let Some(ref limiter) = self.list_buckets_limiter {
return limiter;
}
}
let segments: Vec<&str> = path
.trim_start_matches('/')
.split('/')
.filter(|s| !s.is_empty())
.collect();
if *method == Method::HEAD {
if let Some(ref limiter) = self.head_ops_limiter {
return limiter;
}
}
if segments.len() == 1 {
if let Some(ref limiter) = self.bucket_ops_limiter {
return limiter;
}
} else if segments.len() >= 2 {
if let Some(ref limiter) = self.object_ops_limiter {
return limiter;
}
}
&self.default_limiter
}
}
#[derive(Debug)]
struct FixedWindowLimiter {
setting: RateLimitSetting,
state: Mutex<LimiterState>,
}
#[derive(Debug)]
struct LimiterState {
entries: HashMap<String, LimitEntry>,
last_sweep: Instant,
}
#[derive(Debug, Clone, Copy)]
struct LimitEntry {
window_started: Instant,
count: u32,
}
const SWEEP_MIN_INTERVAL: Duration = Duration::from_secs(60);
const SWEEP_ENTRY_THRESHOLD: usize = 1024;
impl FixedWindowLimiter {
fn new(setting: RateLimitSetting) -> Self {
Self {
setting,
state: Mutex::new(LimiterState {
entries: HashMap::new(),
last_sweep: Instant::now(),
}),
}
}
fn check(&self, key: &str) -> Result<(), u64> {
let now = Instant::now();
let window = Duration::from_secs(self.setting.window_seconds.max(1));
let mut state = self.state.lock();
if state.entries.len() >= SWEEP_ENTRY_THRESHOLD
&& now.duration_since(state.last_sweep) >= SWEEP_MIN_INTERVAL
{
state
.entries
.retain(|_, entry| now.duration_since(entry.window_started) < window);
state.last_sweep = now;
}
let entry = state.entries.entry(key.to_string()).or_insert(LimitEntry {
window_started: now,
count: 0,
});
if now.duration_since(entry.window_started) >= window {
entry.window_started = now;
entry.count = 0;
}
if entry.count >= self.setting.max_requests {
let elapsed = now.duration_since(entry.window_started);
let retry_after = window.saturating_sub(elapsed).as_secs().max(1);
return Err(retry_after);
}
entry.count += 1;
Ok(())
}
}
pub async fn rate_limit_layer(
State(state): State<RateLimitLayerState>,
req: Request,
next: Next,
) -> Response {
let key = rate_limit_key(&req, state.num_trusted_proxies);
let limiter = state.select_limiter(&req);
match limiter.check(&key) {
Ok(()) => next.run(req).await,
Err(retry_after) => {
let resource = req.uri().path().to_string();
too_many_requests(retry_after, &resource)
}
}
}
fn too_many_requests(retry_after: u64, resource: &str) -> Response {
let request_id = uuid::Uuid::new_v4().simple().to_string();
let body = myfsio_xml::response::rate_limit_exceeded_xml(resource, &request_id);
let mut response = (
StatusCode::SERVICE_UNAVAILABLE,
[
(header::CONTENT_TYPE, "application/xml".to_string()),
(header::RETRY_AFTER, retry_after.to_string()),
],
body,
)
.into_response();
if let Ok(value) = request_id.parse() {
response
.headers_mut()
.insert("x-amz-request-id", value);
}
response
}
fn rate_limit_key(req: &Request, num_trusted_proxies: usize) -> String {
format!("ip:{}", client_ip(req, num_trusted_proxies))
}
fn client_ip(req: &Request, num_trusted_proxies: usize) -> String {
if num_trusted_proxies > 0 {
if let Some(value) = req
.headers()
.get("x-forwarded-for")
.and_then(|v| v.to_str().ok())
{
let parts = value
.split(',')
.map(|part| part.trim())
.filter(|part| !part.is_empty())
.collect::<Vec<_>>();
if parts.len() > num_trusted_proxies {
let index = parts.len() - num_trusted_proxies - 1;
return parts[index].to_string();
}
}
if let Some(value) = req.headers().get("x-real-ip").and_then(|v| v.to_str().ok()) {
if !value.trim().is_empty() {
return value.trim().to_string();
}
}
}
req.extensions()
.get::<ConnectInfo<SocketAddr>>()
.map(|ConnectInfo(addr)| addr.ip().to_string())
.unwrap_or_else(|| "unknown".to_string())
}
#[cfg(test)]
mod tests {
use super::*;
use axum::body::Body;
#[test]
fn honors_trusted_proxy_count_for_forwarded_for() {
let req = Request::builder()
.header("x-forwarded-for", "198.51.100.1, 10.0.0.1, 10.0.0.2")
.body(Body::empty())
.unwrap();
assert_eq!(rate_limit_key(&req, 2), "ip:198.51.100.1");
assert_eq!(rate_limit_key(&req, 1), "ip:10.0.0.1");
}
#[test]
fn falls_back_to_connect_info_when_forwarded_for_has_too_few_hops() {
let mut req = Request::builder()
.header("x-forwarded-for", "198.51.100.1")
.body(Body::empty())
.unwrap();
req.extensions_mut()
.insert(ConnectInfo(SocketAddr::from(([203, 0, 113, 9], 443))));
assert_eq!(rate_limit_key(&req, 2), "ip:203.0.113.9");
}
#[test]
fn ignores_forwarded_headers_when_no_proxies_are_trusted() {
let mut req = Request::builder()
.header("x-forwarded-for", "198.51.100.1")
.header("x-real-ip", "198.51.100.2")
.body(Body::empty())
.unwrap();
req.extensions_mut()
.insert(ConnectInfo(SocketAddr::from(([203, 0, 113, 9], 443))));
assert_eq!(rate_limit_key(&req, 0), "ip:203.0.113.9");
}
#[test]
fn uses_connect_info_for_direct_clients() {
let mut req = Request::builder().body(Body::empty()).unwrap();
req.extensions_mut()
.insert(ConnectInfo(SocketAddr::from(([203, 0, 113, 10], 443))));
assert_eq!(rate_limit_key(&req, 0), "ip:203.0.113.10");
}
#[test]
fn fixed_window_rejects_after_quota() {
let limiter = FixedWindowLimiter::new(RateLimitSetting::new(2, 60));
assert!(limiter.check("k").is_ok());
assert!(limiter.check("k").is_ok());
assert!(limiter.check("k").is_err());
}
#[test]
fn sweep_removes_expired_entries() {
let limiter = FixedWindowLimiter::new(RateLimitSetting::new(10, 1));
let far_past = Instant::now() - (SWEEP_MIN_INTERVAL + Duration::from_secs(5));
{
let mut state = limiter.state.lock();
for i in 0..(SWEEP_ENTRY_THRESHOLD + 1024) {
state.entries.insert(
format!("stale-{}", i),
LimitEntry {
window_started: far_past,
count: 5,
},
);
}
state.last_sweep = far_past;
}
let seeded = limiter.state.lock().entries.len();
assert_eq!(seeded, SWEEP_ENTRY_THRESHOLD + 1024);
assert!(limiter.check("fresh").is_ok());
let remaining = limiter.state.lock().entries.len();
assert_eq!(
remaining, 1,
"expected sweep to leave only the fresh entry, got {}",
remaining
);
}
}

View File

@@ -90,7 +90,11 @@ pub async fn session_layer(
resp
}
pub async fn csrf_layer(req: Request, next: Next) -> Response {
pub async fn csrf_layer(
State(state): State<crate::state::AppState>,
req: Request,
next: Next,
) -> Response {
const CSRF_HEADER_ALIAS: &str = "x-csrftoken";
let method = req.method().clone();
@@ -169,7 +173,32 @@ pub async fn csrf_layer(req: Request, next: Next) -> Response {
header_present = header_token.is_some(),
"CSRF token mismatch"
);
(StatusCode::FORBIDDEN, "Invalid CSRF token").into_response()
let accept = parts
.headers
.get(header::ACCEPT)
.and_then(|v| v.to_str().ok())
.unwrap_or("");
let is_form_submit = content_type.starts_with("application/x-www-form-urlencoded")
|| content_type.starts_with("multipart/form-data");
let wants_json =
accept.contains("application/json") || content_type.starts_with("application/json");
if is_form_submit && !wants_json {
let ctx = crate::handlers::ui::base_context(&handle, None);
let mut resp = crate::handlers::ui::render(&state, "csrf_error.html", &ctx);
*resp.status_mut() = StatusCode::FORBIDDEN;
return resp;
}
let mut resp = (
StatusCode::FORBIDDEN,
[(header::CONTENT_TYPE, "application/json")],
r#"{"error":"Invalid CSRF token"}"#,
)
.into_response();
*resp.status_mut() = StatusCode::FORBIDDEN;
resp
}
fn extract_multipart_token(content_type: &str, body: &[u8]) -> Option<String> {

View File

@@ -0,0 +1,107 @@
use axum::body::Body;
use bytes::Bytes;
use http_body::{Body as HttpBody, Frame};
use sha2::{Digest, Sha256};
use std::error::Error;
use std::fmt;
use std::pin::Pin;
use std::task::{Context, Poll};
#[derive(Debug)]
struct Sha256MismatchError {
expected: String,
computed: String,
}
impl Sha256MismatchError {
fn message(&self) -> String {
format!(
"The x-amz-content-sha256 you specified did not match what we received (expected {}, computed {})",
self.expected, self.computed
)
}
}
impl fmt::Display for Sha256MismatchError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"XAmzContentSHA256Mismatch: expected {}, computed {}",
self.expected, self.computed
)
}
}
impl Error for Sha256MismatchError {}
pub struct Sha256VerifyBody {
inner: Body,
expected: String,
hasher: Option<Sha256>,
}
impl Sha256VerifyBody {
pub fn new(inner: Body, expected_hex: String) -> Self {
Self {
inner,
expected: expected_hex.to_ascii_lowercase(),
hasher: Some(Sha256::new()),
}
}
}
impl HttpBody for Sha256VerifyBody {
type Data = Bytes;
type Error = Box<dyn std::error::Error + Send + Sync>;
fn poll_frame(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
let this = self.as_mut().get_mut();
match Pin::new(&mut this.inner).poll_frame(cx) {
Poll::Pending => Poll::Pending,
Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(Box::new(e)))),
Poll::Ready(Some(Ok(frame))) => {
if let Some(data) = frame.data_ref() {
if let Some(h) = this.hasher.as_mut() {
h.update(data);
}
}
Poll::Ready(Some(Ok(frame)))
}
Poll::Ready(None) => {
if let Some(hasher) = this.hasher.take() {
let computed = hex::encode(hasher.finalize());
if computed != this.expected {
return Poll::Ready(Some(Err(Box::new(Sha256MismatchError {
expected: this.expected.clone(),
computed,
}))));
}
}
Poll::Ready(None)
}
}
}
fn is_end_stream(&self) -> bool {
self.inner.is_end_stream()
}
fn size_hint(&self) -> http_body::SizeHint {
self.inner.size_hint()
}
}
pub fn is_hex_sha256(s: &str) -> bool {
s.len() == 64 && s.bytes().all(|b| b.is_ascii_hexdigit())
}
pub fn sha256_mismatch_message(err: &(dyn Error + 'static)) -> Option<String> {
if let Some(mismatch) = err.downcast_ref::<Sha256MismatchError>() {
return Some(mismatch.message());
}
err.source().and_then(sha256_mismatch_message)
}

View File

@@ -24,6 +24,35 @@ impl Default for GcConfig {
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn dry_run_reports_but_does_not_delete_temp_files() {
let tmp = tempfile::tempdir().unwrap();
let tmp_dir = tmp.path().join(".myfsio.sys").join("tmp");
std::fs::create_dir_all(&tmp_dir).unwrap();
let file_path = tmp_dir.join("stale.tmp");
std::fs::write(&file_path, b"temporary").unwrap();
tokio::time::sleep(std::time::Duration::from_millis(5)).await;
let service = GcService::new(
tmp.path().to_path_buf(),
GcConfig {
temp_file_max_age_hours: 0.0,
dry_run: true,
..GcConfig::default()
},
);
let result = service.run_now(false).await.unwrap();
assert_eq!(result["temp_files_deleted"], 1);
assert!(file_path.exists());
}
}
pub struct GcService {
storage_root: PathBuf,
config: GcConfig,

View File

@@ -15,9 +15,9 @@ use crate::session::SessionStore;
use crate::stores::connections::ConnectionStore;
use crate::templates::TemplateEngine;
use myfsio_auth::iam::IamService;
use myfsio_crypto::encryption::EncryptionService;
use myfsio_crypto::encryption::{EncryptionConfig, EncryptionService};
use myfsio_crypto::kms::KmsService;
use myfsio_storage::fs_backend::FsStorageBackend;
use myfsio_storage::fs_backend::{FsStorageBackend, FsStorageBackendConfig};
#[derive(Clone)]
pub struct AppState {
@@ -42,7 +42,16 @@ pub struct AppState {
impl AppState {
pub fn new(config: ServerConfig) -> Self {
let storage = Arc::new(FsStorageBackend::new(config.storage_root.clone()));
let storage = Arc::new(FsStorageBackend::new_with_config(
config.storage_root.clone(),
FsStorageBackendConfig {
object_key_max_length_bytes: config.object_key_max_length_bytes,
object_cache_max_size: config.object_cache_max_size,
bucket_config_cache_ttl: Duration::from_secs_f64(
config.bucket_config_cache_ttl_seconds,
),
},
));
let iam = Arc::new(IamService::new_with_secret(
config.iam_config_path.clone(),
config.secret_key.clone(),
@@ -51,7 +60,13 @@ impl AppState {
let gc = if config.gc_enabled {
Some(Arc::new(GcService::new(
config.storage_root.clone(),
crate::services::gc::GcConfig::default(),
crate::services::gc::GcConfig {
interval_hours: config.gc_interval_hours,
temp_file_max_age_hours: config.gc_temp_file_max_age_hours,
multipart_max_age_days: config.gc_multipart_max_age_days,
lock_file_max_age_hours: config.gc_lock_file_max_age_hours,
dry_run: config.gc_dry_run,
},
)))
} else {
None
@@ -92,7 +107,22 @@ impl AppState {
None
};
let site_registry = Some(Arc::new(SiteRegistry::new(&config.storage_root)));
let site_registry = {
let registry = SiteRegistry::new(&config.storage_root);
if let (Some(site_id), Some(endpoint)) =
(config.site_id.as_deref(), config.site_endpoint.as_deref())
{
registry.set_local_site(crate::services::site_registry::SiteInfo {
site_id: site_id.to_string(),
endpoint: endpoint.to_string(),
region: config.site_region.clone(),
priority: config.site_priority,
display_name: site_id.to_string(),
created_at: Some(chrono::Utc::now().to_rfc3339()),
});
}
Some(Arc::new(registry))
};
let website_domains = if config.website_hosting_enabled {
Some(Arc::new(WebsiteDomainStore::new(&config.storage_root)))
@@ -132,6 +162,7 @@ impl AppState {
let templates = init_templates(&config.templates_dir);
let access_logging = Arc::new(AccessLoggingService::new(&config.storage_root));
let session_ttl = Duration::from_secs(config.session_lifetime_days.saturating_mul(86_400));
Self {
config,
storage,
@@ -148,7 +179,7 @@ impl AppState {
replication,
site_sync,
templates,
sessions: Arc::new(SessionStore::new(Duration::from_secs(60 * 60 * 12))),
sessions: Arc::new(SessionStore::new(session_ttl)),
access_logging,
}
}
@@ -172,7 +203,13 @@ impl AppState {
let encryption = if config.encryption_enabled {
match myfsio_crypto::kms::load_or_create_master_key(&keys_dir).await {
Ok(master_key) => Some(Arc::new(EncryptionService::new(master_key, kms.clone()))),
Ok(master_key) => Some(Arc::new(EncryptionService::with_config(
master_key,
kms.clone(),
EncryptionConfig {
chunk_size: config.encryption_chunk_size_bytes,
},
))),
Err(e) => {
tracing::error!("Failed to initialize encryption: {}", e);
None

View File

Before

Width:  |  Height:  |  Size: 200 KiB

After

Width:  |  Height:  |  Size: 200 KiB

View File

Before

Width:  |  Height:  |  Size: 872 KiB

After

Width:  |  Height:  |  Size: 872 KiB

View File

@@ -73,16 +73,13 @@
</div>
<p class="text-muted">Build or run the Rust server and launch the API plus web UI from a single process.</p>
<div class="alert alert-light border small mb-3">
Runtime note: MyFSIO now runs from the Rust server in <code>rust/myfsio-engine</code>. For the verified runtime configuration list, use the repository <code>docs.md</code>.
Runtime note: the repository root is the Cargo workspace. For the verified runtime configuration list, use the repository <code>docs.md</code>.
</div>
<ol class="docs-steps">
<li>Install a current Rust toolchain.</li>
<li>Change into <code>rust/myfsio-engine</code>.</li>
<li>Start the server with <code>cargo run -p myfsio-server --</code>.</li>
<li>From the repository root, start the server with <code>cargo run -p myfsio-server --</code>.</li>
</ol>
<pre class="mb-3"><code class="language-bash">cd rust/myfsio-engine
# Run API + UI
<pre class="mb-3"><code class="language-bash"># Run API + UI
cargo run -p myfsio-server --
# Show resolved configuration
@@ -112,7 +109,7 @@ cargo build --release -p myfsio-server
<tbody>
<tr>
<td><code>API_BASE_URL</code></td>
<td><code>http://127.0.0.1:5000</code></td>
<td>Derived from <code>HOST</code>/<code>PORT</code></td>
<td>Internal S3 API URL used by the web UI proxy. Also used for presigned URL generation. Set to your public URL if running behind a reverse proxy.</td>
</tr>
<tr>
@@ -184,33 +181,18 @@ cargo build --release -p myfsio-server
<tr>
<td><code>RATE_LIMIT_DEFAULT</code></td>
<td><code>200 per minute</code></td>
<td>Default API rate limit.</td>
</tr>
<tr>
<td><code>RATE_LIMIT_LIST_BUCKETS</code></td>
<td><code>60 per minute</code></td>
<td>Rate limit for listing buckets.</td>
</tr>
<tr>
<td><code>RATE_LIMIT_BUCKET_OPS</code></td>
<td><code>120 per minute</code></td>
<td>Rate limit for bucket operations.</td>
</tr>
<tr>
<td><code>RATE_LIMIT_OBJECT_OPS</code></td>
<td><code>240 per minute</code></td>
<td>Rate limit for object operations.</td>
</tr>
<tr>
<td><code>RATE_LIMIT_HEAD_OPS</code></td>
<td><code>100 per minute</code></td>
<td>Rate limit for HEAD requests.</td>
<td>Default rate limit for S3 and KMS API endpoints.</td>
</tr>
<tr>
<td><code>RATE_LIMIT_ADMIN</code></td>
<td><code>60 per minute</code></td>
<td>Rate limit for admin API endpoints (<code>/admin/*</code>).</td>
</tr>
<tr>
<td><code>RATE_LIMIT_STORAGE_URI</code></td>
<td><code>memory://</code></td>
<td>Rate limit storage backend. Only in-memory storage is currently supported.</td>
</tr>
<tr>
<td><code>ADMIN_ACCESS_KEY</code></td>
<td>(none)</td>
@@ -377,8 +359,8 @@ cargo build --release -p myfsio-server
</tr>
<tr>
<td><code>NUM_TRUSTED_PROXIES</code></td>
<td><code>1</code></td>
<td>Number of trusted reverse proxies for <code>X-Forwarded-*</code> headers.</td>
<td><code>0</code></td>
<td>Number of trusted reverse proxies for <code>X-Forwarded-*</code> headers. Forwarded IP headers are ignored when this is <code>0</code>.</td>
</tr>
<tr>
<td><code>ALLOWED_REDIRECT_HOSTS</code></td>
@@ -2078,7 +2060,7 @@ curl "{{ api_base | replace(from="/api", to="/ui") }}/metrics/operations/history
<tr>
<td>Large folder uploads hitting rate limits (429)</td>
<td><code>RATE_LIMIT_DEFAULT</code> exceeded (200/min)</td>
<td>Increase rate limit in env config, use Redis backend (<code>RATE_LIMIT_STORAGE_URI=redis://host:port</code>) for distributed setups, or upload in smaller batches.</td>
<td>Increase <code>RATE_LIMIT_DEFAULT</code> in env config or upload in smaller batches. Distributed rate-limit storage is not supported yet.</td>
</tr>
</tbody>
</table>

View File

@@ -133,7 +133,7 @@
{% endif %}
<div class="row g-3">
{% for user in users %}
<div class="col-md-6 col-xl-4 iam-user-item" data-user-id="{{ user.user_id }}" data-access-key="{{ user.access_key }}" data-display-name="{{ user.display_name|lower }}" data-access-key-filter="{{ user.access_key|lower }}">
<div class="col-md-6 col-xl-4 iam-user-item" data-user-id="{{ user.user_id }}" data-access-key="{{ user.access_key }}" data-display-name="{{ user.display_name|lower }}" data-access-key-filter="{{ user.access_key|lower }}" data-update-url="{{ url_for(endpoint="ui.update_iam_user", user_id=user.user_id) }}">
<div class="card h-100 iam-user-card{% if user.is_admin %} iam-admin-card{% else %}{% endif %}">
<div class="card-body">
<div class="d-flex align-items-start justify-content-between mb-3">

View File

@@ -1,5 +1,7 @@
use axum::body::Body;
use axum::http::{Method, Request, StatusCode};
use base64::engine::general_purpose::URL_SAFE;
use base64::Engine;
use http_body_util::BodyExt;
use myfsio_storage::traits::{AsyncReadStream, StorageEngine};
use serde_json::Value;
@@ -53,6 +55,8 @@ fn test_app_with_iam(iam_json: serde_json::Value) -> (axum::Router, tempfile::Te
ui_enabled: false,
templates_dir: std::path::PathBuf::from("templates"),
static_dir: std::path::PathBuf::from("static"),
multipart_min_part_size: 1,
..myfsio_server::config::ServerConfig::default()
};
let state = myfsio_server::state::AppState::new(config);
let app = myfsio_server::create_router(state);
@@ -80,6 +84,124 @@ fn test_app() -> (axum::Router, tempfile::TempDir) {
}))
}
fn test_app_with_rate_limits(
default: myfsio_server::config::RateLimitSetting,
admin: myfsio_server::config::RateLimitSetting,
) -> (axum::Router, tempfile::TempDir) {
let tmp = tempfile::TempDir::new().unwrap();
let iam_path = tmp.path().join(".myfsio.sys").join("config");
std::fs::create_dir_all(&iam_path).unwrap();
std::fs::write(
iam_path.join("iam.json"),
serde_json::json!({
"version": 2,
"users": [{
"user_id": "u-test1234",
"display_name": "admin",
"enabled": true,
"access_keys": [{
"access_key": TEST_ACCESS_KEY,
"secret_key": TEST_SECRET_KEY,
"status": "active"
}],
"policies": [{
"bucket": "*",
"actions": ["*"],
"prefix": "*"
}]
}]
})
.to_string(),
)
.unwrap();
let config = myfsio_server::config::ServerConfig {
bind_addr: "127.0.0.1:0".parse().unwrap(),
ui_bind_addr: "127.0.0.1:0".parse().unwrap(),
storage_root: tmp.path().to_path_buf(),
iam_config_path: iam_path.join("iam.json"),
ratelimit_default: default,
ratelimit_list_buckets: default,
ratelimit_bucket_ops: default,
ratelimit_object_ops: default,
ratelimit_head_ops: default,
ratelimit_admin: admin,
ui_enabled: false,
..myfsio_server::config::ServerConfig::default()
};
let state = myfsio_server::state::AppState::new(config);
let app = myfsio_server::create_router(state);
(app, tmp)
}
#[tokio::test]
async fn rate_limit_default_and_admin_are_independent() {
let (app, _tmp) = test_app_with_rate_limits(
myfsio_server::config::RateLimitSetting::new(1, 60),
myfsio_server::config::RateLimitSetting::new(2, 60),
);
let first = app
.clone()
.oneshot(
Request::builder()
.uri("/myfsio/health")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(first.status(), StatusCode::OK);
let second = app
.clone()
.oneshot(
Request::builder()
.uri("/myfsio/health")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(second.status(), StatusCode::SERVICE_UNAVAILABLE);
assert!(second.headers().contains_key("retry-after"));
let admin_first = app
.clone()
.oneshot(
Request::builder()
.uri("/admin/gc/status")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(admin_first.status(), StatusCode::FORBIDDEN);
let admin_second = app
.clone()
.oneshot(
Request::builder()
.uri("/admin/gc/status")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(admin_second.status(), StatusCode::FORBIDDEN);
let admin_third = app
.oneshot(
Request::builder()
.uri("/admin/gc/status")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(admin_third.status(), StatusCode::SERVICE_UNAVAILABLE);
}
fn test_ui_state() -> (myfsio_server::state::AppState, tempfile::TempDir) {
let tmp = tempfile::TempDir::new().unwrap();
let iam_path = tmp.path().join(".myfsio.sys").join("config");
@@ -147,6 +269,7 @@ fn test_ui_state() -> (myfsio_server::state::AppState, tempfile::TempDir) {
ui_enabled: true,
templates_dir: manifest_dir.join("templates"),
static_dir: manifest_dir.join("static"),
..myfsio_server::config::ServerConfig::default()
};
(myfsio_server::state::AppState::new(config), tmp)
}
@@ -303,6 +426,7 @@ fn test_website_state() -> (myfsio_server::state::AppState, tempfile::TempDir) {
ui_enabled: false,
templates_dir: std::path::PathBuf::from("templates"),
static_dir: std::path::PathBuf::from("static"),
..myfsio_server::config::ServerConfig::default()
};
(myfsio_server::state::AppState::new(config), tmp)
}
@@ -1082,7 +1206,7 @@ async fn test_ui_metrics_history_endpoint_reads_system_history() {
config_root.join("metrics_history.json"),
serde_json::json!({
"history": [{
"timestamp": "2026-04-20T00:00:00Z",
"timestamp": chrono::Utc::now().to_rfc3339(),
"cpu_percent": 12.5,
"memory_percent": 33.3,
"disk_percent": 44.4,
@@ -1131,6 +1255,7 @@ async fn test_ui_metrics_history_endpoint_reads_system_history() {
ui_enabled: true,
templates_dir: manifest_dir.join("templates"),
static_dir: manifest_dir.join("static"),
..myfsio_server::config::ServerConfig::default()
};
let state = myfsio_server::state::AppState::new(config);
let (session_id, _csrf) = authenticated_ui_session(&state);
@@ -2186,9 +2311,16 @@ async fn test_versioned_object_can_be_read_and_deleted_by_version_id() {
)
.unwrap();
let archived_version_id = list_body
.split("<Version>")
.skip(1)
.find(|block| block.contains("<IsLatest>false</IsLatest>"))
.and_then(|block| {
block
.split("<VersionId>")
.filter_map(|part| part.split_once("</VersionId>").map(|(id, _)| id))
.find(|id| *id != "null")
.nth(1)
.and_then(|s| s.split_once("</VersionId>").map(|(id, _)| id))
})
.filter(|id| *id != "null")
.expect("archived version id")
.to_string();
@@ -2276,6 +2408,457 @@ async fn test_versioned_object_can_be_read_and_deleted_by_version_id() {
assert_eq!(missing_resp.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn test_versioned_put_and_delete_emit_version_headers_and_delete_markers() {
let (app, _tmp) = test_app();
app.clone()
.oneshot(signed_request(Method::PUT, "/compat-bucket", Body::empty()))
.await
.unwrap();
app.clone()
.oneshot(
Request::builder()
.method(Method::PUT)
.uri("/compat-bucket?versioning")
.header("x-access-key", TEST_ACCESS_KEY)
.header("x-secret-key", TEST_SECRET_KEY)
.body(Body::from(
"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>",
))
.unwrap(),
)
.await
.unwrap();
let put_resp = app
.clone()
.oneshot(signed_request(
Method::PUT,
"/compat-bucket/doc.txt",
Body::from("first"),
))
.await
.unwrap();
assert_eq!(put_resp.status(), StatusCode::OK);
let first_version = put_resp
.headers()
.get("x-amz-version-id")
.expect("PUT on versioned bucket must emit x-amz-version-id")
.to_str()
.unwrap()
.to_string();
assert!(!first_version.is_empty());
let overwrite_resp = app
.clone()
.oneshot(signed_request(
Method::PUT,
"/compat-bucket/doc.txt",
Body::from("second"),
))
.await
.unwrap();
assert_eq!(overwrite_resp.status(), StatusCode::OK);
let second_version = overwrite_resp
.headers()
.get("x-amz-version-id")
.expect("overwrite on versioned bucket must emit a new x-amz-version-id")
.to_str()
.unwrap()
.to_string();
assert_ne!(first_version, second_version);
let delete_resp = app
.clone()
.oneshot(signed_request(
Method::DELETE,
"/compat-bucket/doc.txt",
Body::empty(),
))
.await
.unwrap();
assert_eq!(delete_resp.status(), StatusCode::NO_CONTENT);
assert_eq!(
delete_resp
.headers()
.get("x-amz-delete-marker")
.and_then(|v| v.to_str().ok()),
Some("true")
);
assert!(delete_resp.headers().contains_key("x-amz-version-id"));
let versions_resp = app
.oneshot(signed_request(
Method::GET,
"/compat-bucket?versions",
Body::empty(),
))
.await
.unwrap();
let versions_body = String::from_utf8(
versions_resp
.into_body()
.collect()
.await
.unwrap()
.to_bytes()
.to_vec(),
)
.unwrap();
assert!(
versions_body.contains("<DeleteMarker>"),
"expected DeleteMarker entry in ListObjectVersions output, got: {}",
versions_body
);
}
#[tokio::test]
async fn test_consecutive_slashes_in_key_round_trip() {
let (app, _tmp) = test_app();
app.clone()
.oneshot(signed_request(Method::PUT, "/slashes-bucket", Body::empty()))
.await
.unwrap();
let put_ab = app
.clone()
.oneshot(signed_request(
Method::PUT,
"/slashes-bucket/a/b",
Body::from("single"),
))
.await
.unwrap();
assert_eq!(put_ab.status(), StatusCode::OK);
let put_double = app
.clone()
.oneshot(signed_request(
Method::PUT,
"/slashes-bucket/a//b",
Body::from("double"),
))
.await
.unwrap();
assert_eq!(put_double.status(), StatusCode::OK);
let put_triple = app
.clone()
.oneshot(signed_request(
Method::PUT,
"/slashes-bucket/a///b",
Body::from("triple"),
))
.await
.unwrap();
assert_eq!(put_triple.status(), StatusCode::OK);
let get_ab = app
.clone()
.oneshot(signed_request(
Method::GET,
"/slashes-bucket/a/b",
Body::empty(),
))
.await
.unwrap();
assert_eq!(get_ab.status(), StatusCode::OK);
let body_ab = get_ab.into_body().collect().await.unwrap().to_bytes();
assert_eq!(&body_ab[..], b"single");
let get_triple = app
.clone()
.oneshot(signed_request(
Method::GET,
"/slashes-bucket/a///b",
Body::empty(),
))
.await
.unwrap();
assert_eq!(get_triple.status(), StatusCode::OK);
let body_triple = get_triple.into_body().collect().await.unwrap().to_bytes();
assert_eq!(&body_triple[..], b"triple");
let list_resp = app
.oneshot(signed_request(
Method::GET,
"/slashes-bucket?list-type=2",
Body::empty(),
))
.await
.unwrap();
let list_body = String::from_utf8(
list_resp
.into_body()
.collect()
.await
.unwrap()
.to_bytes()
.to_vec(),
)
.unwrap();
assert!(
list_body.contains("<Key>a/b</Key>"),
"expected a/b in listing: {}",
list_body
);
assert!(
list_body.contains("<Key>a//b</Key>"),
"expected a//b in listing: {}",
list_body
);
assert!(
list_body.contains("<Key>a///b</Key>"),
"expected a///b in listing: {}",
list_body
);
}
#[tokio::test]
async fn test_delete_live_version_restores_previous_to_live_slot() {
let (app, _tmp) = test_app();
app.clone()
.oneshot(signed_request(Method::PUT, "/restore-bucket", Body::empty()))
.await
.unwrap();
app.clone()
.oneshot(
Request::builder()
.method(Method::PUT)
.uri("/restore-bucket?versioning")
.header("x-access-key", TEST_ACCESS_KEY)
.header("x-secret-key", TEST_SECRET_KEY)
.body(Body::from(
"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>",
))
.unwrap(),
)
.await
.unwrap();
let v1_resp = app
.clone()
.oneshot(signed_request(
Method::PUT,
"/restore-bucket/k",
Body::from("one"),
))
.await
.unwrap();
let v1 = v1_resp
.headers()
.get("x-amz-version-id")
.unwrap()
.to_str()
.unwrap()
.to_string();
let v2_resp = app
.clone()
.oneshot(signed_request(
Method::PUT,
"/restore-bucket/k",
Body::from("two"),
))
.await
.unwrap();
let v2 = v2_resp
.headers()
.get("x-amz-version-id")
.unwrap()
.to_str()
.unwrap()
.to_string();
assert_ne!(v1, v2);
let del = app
.clone()
.oneshot(signed_request(
Method::DELETE,
&format!("/restore-bucket/k?versionId={}", v2),
Body::empty(),
))
.await
.unwrap();
assert_eq!(del.status(), StatusCode::NO_CONTENT);
let get_live = app
.clone()
.oneshot(signed_request(
Method::GET,
"/restore-bucket/k",
Body::empty(),
))
.await
.unwrap();
assert_eq!(get_live.status(), StatusCode::OK);
let body = get_live.into_body().collect().await.unwrap().to_bytes();
assert_eq!(&body[..], b"one");
let get_v1 = app
.oneshot(signed_request(
Method::GET,
&format!("/restore-bucket/k?versionId={}", v1),
Body::empty(),
))
.await
.unwrap();
assert_eq!(get_v1.status(), StatusCode::OK);
}
#[tokio::test]
async fn test_delete_active_delete_marker_restores_previous_to_live_slot() {
let (app, _tmp) = test_app();
app.clone()
.oneshot(signed_request(Method::PUT, "/undel-bucket", Body::empty()))
.await
.unwrap();
app.clone()
.oneshot(
Request::builder()
.method(Method::PUT)
.uri("/undel-bucket?versioning")
.header("x-access-key", TEST_ACCESS_KEY)
.header("x-secret-key", TEST_SECRET_KEY)
.body(Body::from(
"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>",
))
.unwrap(),
)
.await
.unwrap();
app.clone()
.oneshot(signed_request(
Method::PUT,
"/undel-bucket/k",
Body::from("only"),
))
.await
.unwrap();
let del = app
.clone()
.oneshot(signed_request(
Method::DELETE,
"/undel-bucket/k",
Body::empty(),
))
.await
.unwrap();
let dm_version = del
.headers()
.get("x-amz-version-id")
.unwrap()
.to_str()
.unwrap()
.to_string();
assert_eq!(
del.headers()
.get("x-amz-delete-marker")
.and_then(|v| v.to_str().ok()),
Some("true")
);
let shadowed = app
.clone()
.oneshot(signed_request(
Method::GET,
"/undel-bucket/k",
Body::empty(),
))
.await
.unwrap();
assert_eq!(shadowed.status(), StatusCode::NOT_FOUND);
let del_dm = app
.clone()
.oneshot(signed_request(
Method::DELETE,
&format!("/undel-bucket/k?versionId={}", dm_version),
Body::empty(),
))
.await
.unwrap();
assert_eq!(del_dm.status(), StatusCode::NO_CONTENT);
let restored = app
.oneshot(signed_request(
Method::GET,
"/undel-bucket/k",
Body::empty(),
))
.await
.unwrap();
assert_eq!(restored.status(), StatusCode::OK);
let body = restored.into_body().collect().await.unwrap().to_bytes();
assert_eq!(&body[..], b"only");
}
#[tokio::test]
async fn test_versioned_get_on_delete_marker_returns_method_not_allowed() {
let (app, _tmp) = test_app();
app.clone()
.oneshot(signed_request(Method::PUT, "/dm-bucket", Body::empty()))
.await
.unwrap();
app.clone()
.oneshot(
Request::builder()
.method(Method::PUT)
.uri("/dm-bucket?versioning")
.header("x-access-key", TEST_ACCESS_KEY)
.header("x-secret-key", TEST_SECRET_KEY)
.body(Body::from(
"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>",
))
.unwrap(),
)
.await
.unwrap();
app.clone()
.oneshot(signed_request(
Method::PUT,
"/dm-bucket/k",
Body::from("x"),
))
.await
.unwrap();
let del = app
.clone()
.oneshot(signed_request(
Method::DELETE,
"/dm-bucket/k",
Body::empty(),
))
.await
.unwrap();
let dm_version = del
.headers()
.get("x-amz-version-id")
.unwrap()
.to_str()
.unwrap()
.to_string();
let versioned = app
.oneshot(signed_request(
Method::GET,
&format!("/dm-bucket/k?versionId={}", dm_version),
Body::empty(),
))
.await
.unwrap();
assert_eq!(versioned.status(), StatusCode::METHOD_NOT_ALLOWED);
}
#[tokio::test]
async fn test_retention_is_enforced_when_deleting_archived_version() {
let (app, _tmp) = test_app();
@@ -2356,9 +2939,16 @@ async fn test_retention_is_enforced_when_deleting_archived_version() {
)
.unwrap();
let archived_version_id = list_body
.split("<Version>")
.skip(1)
.find(|block| block.contains("<IsLatest>false</IsLatest>"))
.and_then(|block| {
block
.split("<VersionId>")
.filter_map(|part| part.split_once("</VersionId>").map(|(id, _)| id))
.find(|id| *id != "null")
.nth(1)
.and_then(|s| s.split_once("</VersionId>").map(|(id, _)| id))
})
.filter(|id| *id != "null")
.expect("archived version id")
.to_string();
@@ -2444,6 +3034,132 @@ async fn test_put_object_validates_content_md5() {
assert_eq!(good_resp.status(), StatusCode::OK);
}
#[tokio::test]
async fn test_x_amz_content_sha256_mismatch_returns_bad_digest() {
let (app, _tmp) = test_app();
app.clone()
.oneshot(signed_request(Method::PUT, "/sha256-bucket", Body::empty()))
.await
.unwrap();
let bad_resp = app
.oneshot(
Request::builder()
.method(Method::PUT)
.uri("/sha256-bucket/object.txt")
.header("x-access-key", TEST_ACCESS_KEY)
.header("x-secret-key", TEST_SECRET_KEY)
.header(
"x-amz-content-sha256",
"0000000000000000000000000000000000000000000000000000000000000000",
)
.body(Body::from("hello"))
.unwrap(),
)
.await
.unwrap();
assert_eq!(bad_resp.status(), StatusCode::BAD_REQUEST);
let bad_body = String::from_utf8(
bad_resp
.into_body()
.collect()
.await
.unwrap()
.to_bytes()
.to_vec(),
)
.unwrap();
assert!(bad_body.contains("<Code>BadDigest</Code>"));
assert!(bad_body.contains("x-amz-content-sha256"));
}
#[tokio::test]
async fn test_max_keys_zero_respects_marker_and_v2_cursors() {
let (app, _tmp) = test_app();
app.clone()
.oneshot(signed_request(Method::PUT, "/cursor-bucket", Body::empty()))
.await
.unwrap();
for key in ["a.txt", "b.txt"] {
app.clone()
.oneshot(signed_request(
Method::PUT,
&format!("/cursor-bucket/{}", key),
Body::from(key.to_string()),
))
.await
.unwrap();
}
let marker_resp = app
.clone()
.oneshot(signed_request(
Method::GET,
"/cursor-bucket?max-keys=0&marker=b.txt",
Body::empty(),
))
.await
.unwrap();
let marker_body = String::from_utf8(
marker_resp
.into_body()
.collect()
.await
.unwrap()
.to_bytes()
.to_vec(),
)
.unwrap();
assert!(marker_body.contains("<IsTruncated>false</IsTruncated>"));
let start_after_resp = app
.clone()
.oneshot(signed_request(
Method::GET,
"/cursor-bucket?list-type=2&max-keys=0&start-after=b.txt",
Body::empty(),
))
.await
.unwrap();
let start_after_body = String::from_utf8(
start_after_resp
.into_body()
.collect()
.await
.unwrap()
.to_bytes()
.to_vec(),
)
.unwrap();
assert!(start_after_body.contains("<IsTruncated>false</IsTruncated>"));
let token = URL_SAFE.encode("b.txt");
let token_resp = app
.oneshot(signed_request(
Method::GET,
&format!(
"/cursor-bucket?list-type=2&max-keys=0&continuation-token={}",
token
),
Body::empty(),
))
.await
.unwrap();
let token_body = String::from_utf8(
token_resp
.into_body()
.collect()
.await
.unwrap()
.to_bytes()
.to_vec(),
)
.unwrap();
assert!(token_body.contains("<IsTruncated>false</IsTruncated>"));
}
#[tokio::test]
async fn test_put_object_tagging_and_standard_headers_are_persisted() {
let (app, _tmp) = test_app();
@@ -3851,6 +4567,7 @@ async fn test_non_admin_authorization_enforced() {
ui_enabled: false,
templates_dir: std::path::PathBuf::from("templates"),
static_dir: std::path::PathBuf::from("static"),
..myfsio_server::config::ServerConfig::default()
};
let state = myfsio_server::state::AppState::new(config);
state.storage.create_bucket("authz-bucket").await.unwrap();
@@ -3932,6 +4649,7 @@ async fn test_app_encrypted() -> (axum::Router, tempfile::TempDir) {
ui_enabled: false,
templates_dir: std::path::PathBuf::from("templates"),
static_dir: std::path::PathBuf::from("static"),
..myfsio_server::config::ServerConfig::default()
};
let state = myfsio_server::state::AppState::new_with_encryption(config).await;
let app = myfsio_server::create_router(state);

View File

@@ -17,10 +17,18 @@ pub enum StorageError {
key: String,
version_id: String,
},
#[error("Object is a delete marker: {bucket}/{key}")]
DeleteMarker {
bucket: String,
key: String,
version_id: String,
},
#[error("Invalid bucket name: {0}")]
InvalidBucketName(String),
#[error("Invalid object key: {0}")]
InvalidObjectKey(String),
#[error("Method not allowed: {0}")]
MethodNotAllowed(String),
#[error("Upload not found: {0}")]
UploadNotFound(String),
#[error("Quota exceeded: {0}")]
@@ -42,7 +50,7 @@ impl From<StorageError> for S3Error {
S3Error::from_code(S3ErrorCode::NoSuchBucket).with_resource(format!("/{}", name))
}
StorageError::BucketAlreadyExists(name) => {
S3Error::from_code(S3ErrorCode::BucketAlreadyExists)
S3Error::from_code(S3ErrorCode::BucketAlreadyOwnedByYou)
.with_resource(format!("/{}", name))
}
StorageError::BucketNotEmpty(name) => {
@@ -58,10 +66,17 @@ impl From<StorageError> for S3Error {
version_id,
} => S3Error::from_code(S3ErrorCode::NoSuchVersion)
.with_resource(format!("/{}/{}?versionId={}", bucket, key, version_id)),
StorageError::DeleteMarker {
bucket,
key,
version_id,
} => S3Error::from_code(S3ErrorCode::MethodNotAllowed)
.with_resource(format!("/{}/{}?versionId={}", bucket, key, version_id)),
StorageError::InvalidBucketName(msg) => {
S3Error::new(S3ErrorCode::InvalidBucketName, msg)
}
StorageError::InvalidObjectKey(msg) => S3Error::new(S3ErrorCode::InvalidKey, msg),
StorageError::MethodNotAllowed(msg) => S3Error::new(S3ErrorCode::MethodNotAllowed, msg),
StorageError::UploadNotFound(id) => S3Error::new(
S3ErrorCode::NoSuchUpload,
format!("Upload {} not found", id),

View File

@@ -62,14 +62,14 @@ pub trait StorageEngine: Send + Sync {
version_id: &str,
) -> StorageResult<HashMap<String, String>>;
async fn delete_object(&self, bucket: &str, key: &str) -> StorageResult<()>;
async fn delete_object(&self, bucket: &str, key: &str) -> StorageResult<DeleteOutcome>;
async fn delete_object_version(
&self,
bucket: &str,
key: &str,
version_id: &str,
) -> StorageResult<()>;
) -> StorageResult<DeleteOutcome>;
async fn copy_object(
&self,

View File

@@ -47,6 +47,7 @@ pub fn validate_object_key(
normalized.split('/').collect()
};
for part in &parts {
if part.is_empty() {
continue;
@@ -60,6 +61,12 @@ pub fn validate_object_key(
return Some("Object key contains invalid segments".to_string());
}
if part.len() > 255 {
return Some(
"Object key contains a path segment that exceeds 255 bytes".to_string(),
);
}
if part.chars().any(|c| (c as u32) < 32) {
return Some("Object key contains control characters".to_string());
}
@@ -98,6 +105,15 @@ pub fn validate_object_key(
}
}
for part in &non_empty_parts {
if *part == ".__myfsio_dirobj__"
|| *part == ".__myfsio_empty__"
|| part.starts_with("_index.json")
{
return Some("Object key segment uses a reserved internal name".to_string());
}
}
None
}
@@ -132,6 +148,13 @@ pub fn validate_bucket_name(bucket_name: &str) -> Option<String> {
return Some("Bucket name must not be formatted as an IP address".to_string());
}
if bucket_name.starts_with("xn--") {
return Some("Bucket name must not start with the reserved prefix 'xn--'".to_string());
}
if bucket_name.ends_with("-s3alias") || bucket_name.ends_with("--ol-s3") {
return Some("Bucket name must not end with a reserved suffix".to_string());
}
None
}

View File

@@ -8,3 +8,4 @@ myfsio-common = { path = "../myfsio-common" }
quick-xml = { workspace = true }
serde = { workspace = true }
chrono = { workspace = true }
percent-encoding = { workspace = true }

View File

@@ -1,13 +1,13 @@
use quick_xml::events::Event;
use quick_xml::Reader;
#[derive(Debug, Default)]
#[derive(Debug, Default, Clone)]
pub struct DeleteObjectsRequest {
pub objects: Vec<ObjectIdentifier>,
pub quiet: bool,
}
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct ObjectIdentifier {
pub key: String,
pub version_id: Option<String>,
@@ -86,6 +86,11 @@ pub fn parse_complete_multipart_upload(xml: &str) -> Result<CompleteMultipartUpl
}
pub fn parse_delete_objects(xml: &str) -> Result<DeleteObjectsRequest, String> {
let trimmed = xml.trim();
if trimmed.is_empty() {
return Err("Request body is empty".to_string());
}
let mut reader = Reader::from_str(xml);
let mut result = DeleteObjectsRequest::default();
let mut buf = Vec::new();
@@ -93,18 +98,43 @@ pub fn parse_delete_objects(xml: &str) -> Result<DeleteObjectsRequest, String> {
let mut current_key: Option<String> = None;
let mut current_version_id: Option<String> = None;
let mut in_object = false;
let mut saw_delete_root = false;
let mut first_element_seen = false;
loop {
match reader.read_event_into(&mut buf) {
let event = reader.read_event_into(&mut buf);
match event {
Ok(Event::Start(ref e)) => {
let name = String::from_utf8_lossy(e.name().as_ref()).to_string();
current_tag = name.clone();
if name == "Object" {
if !first_element_seen {
first_element_seen = true;
if name != "Delete" {
return Err(format!(
"Expected <Delete> root element, found <{}>",
name
));
}
saw_delete_root = true;
} else if name == "Object" {
in_object = true;
current_key = None;
current_version_id = None;
}
}
Ok(Event::Empty(ref e)) => {
let name = String::from_utf8_lossy(e.name().as_ref()).to_string();
if !first_element_seen {
first_element_seen = true;
if name != "Delete" {
return Err(format!(
"Expected <Delete> root element, found <{}>",
name
));
}
saw_delete_root = true;
}
}
Ok(Event::Text(ref e)) => {
let text = e.unescape().map_err(|e| e.to_string())?.to_string();
match current_tag.as_str() {
@@ -139,6 +169,13 @@ pub fn parse_delete_objects(xml: &str) -> Result<DeleteObjectsRequest, String> {
buf.clear();
}
if !saw_delete_root {
return Err("Expected <Delete> root element".to_string());
}
if result.objects.is_empty() {
return Err("Delete request must contain at least one <Object>".to_string());
}
Ok(result)
}

View File

@@ -8,6 +8,23 @@ pub fn format_s3_datetime(dt: &DateTime<Utc>) -> String {
dt.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()
}
pub fn rate_limit_exceeded_xml(resource: &str, request_id: &str) -> String {
format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<Error><Code>SlowDown</Code><Message>Please reduce your request rate</Message><Resource>{}</Resource><RequestId>{}</RequestId></Error>",
xml_escape(resource),
xml_escape(request_id),
)
}
fn xml_escape(s: &str) -> String {
s.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('>', "&gt;")
.replace('"', "&quot;")
.replace('\'', "&apos;")
}
pub fn list_buckets_xml(owner_id: &str, owner_name: &str, buckets: &[BucketMeta]) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
@@ -56,6 +73,21 @@ pub fn list_buckets_xml(owner_id: &str, owner_name: &str, buckets: &[BucketMeta]
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
fn maybe_url_encode(value: &str, encoding_type: Option<&str>) -> String {
if matches!(encoding_type, Some(v) if v.eq_ignore_ascii_case("url")) {
percent_encoding::utf8_percent_encode(value, KEY_ENCODE_SET).to_string()
} else {
value.to_string()
}
}
const KEY_ENCODE_SET: &percent_encoding::AsciiSet = &percent_encoding::NON_ALPHANUMERIC
.remove(b'-')
.remove(b'_')
.remove(b'.')
.remove(b'~')
.remove(b'/');
pub fn list_objects_v2_xml(
bucket_name: &str,
prefix: &str,
@@ -67,6 +99,34 @@ pub fn list_objects_v2_xml(
continuation_token: Option<&str>,
next_continuation_token: Option<&str>,
key_count: usize,
) -> String {
list_objects_v2_xml_with_encoding(
bucket_name,
prefix,
delimiter,
max_keys,
objects,
common_prefixes,
is_truncated,
continuation_token,
next_continuation_token,
key_count,
None,
)
}
pub fn list_objects_v2_xml_with_encoding(
bucket_name: &str,
prefix: &str,
delimiter: &str,
max_keys: usize,
objects: &[ObjectMeta],
common_prefixes: &[String],
is_truncated: bool,
continuation_token: Option<&str>,
next_continuation_token: Option<&str>,
key_count: usize,
encoding_type: Option<&str>,
) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
@@ -79,13 +139,22 @@ pub fn list_objects_v2_xml(
writer.write_event(Event::Start(start)).unwrap();
write_text_element(&mut writer, "Name", bucket_name);
write_text_element(&mut writer, "Prefix", prefix);
write_text_element(&mut writer, "Prefix", &maybe_url_encode(prefix, encoding_type));
if !delimiter.is_empty() {
write_text_element(&mut writer, "Delimiter", delimiter);
write_text_element(
&mut writer,
"Delimiter",
&maybe_url_encode(delimiter, encoding_type),
);
}
write_text_element(&mut writer, "MaxKeys", &max_keys.to_string());
write_text_element(&mut writer, "KeyCount", &key_count.to_string());
write_text_element(&mut writer, "IsTruncated", &is_truncated.to_string());
if let Some(encoding) = encoding_type {
if !encoding.is_empty() {
write_text_element(&mut writer, "EncodingType", encoding);
}
}
if let Some(token) = continuation_token {
write_text_element(&mut writer, "ContinuationToken", token);
@@ -98,7 +167,7 @@ pub fn list_objects_v2_xml(
writer
.write_event(Event::Start(BytesStart::new("Contents")))
.unwrap();
write_text_element(&mut writer, "Key", &obj.key);
write_text_element(&mut writer, "Key", &maybe_url_encode(&obj.key, encoding_type));
write_text_element(
&mut writer,
"LastModified",
@@ -122,7 +191,7 @@ pub fn list_objects_v2_xml(
writer
.write_event(Event::Start(BytesStart::new("CommonPrefixes")))
.unwrap();
write_text_element(&mut writer, "Prefix", prefix);
write_text_element(&mut writer, "Prefix", &maybe_url_encode(prefix, encoding_type));
writer
.write_event(Event::End(BytesEnd::new("CommonPrefixes")))
.unwrap();
@@ -145,6 +214,32 @@ pub fn list_objects_v1_xml(
common_prefixes: &[String],
is_truncated: bool,
next_marker: Option<&str>,
) -> String {
list_objects_v1_xml_with_encoding(
bucket_name,
prefix,
marker,
delimiter,
max_keys,
objects,
common_prefixes,
is_truncated,
next_marker,
None,
)
}
pub fn list_objects_v1_xml_with_encoding(
bucket_name: &str,
prefix: &str,
marker: &str,
delimiter: &str,
max_keys: usize,
objects: &[ObjectMeta],
common_prefixes: &[String],
is_truncated: bool,
next_marker: Option<&str>,
encoding_type: Option<&str>,
) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
@@ -157,27 +252,36 @@ pub fn list_objects_v1_xml(
writer.write_event(Event::Start(start)).unwrap();
write_text_element(&mut writer, "Name", bucket_name);
write_text_element(&mut writer, "Prefix", prefix);
write_text_element(&mut writer, "Marker", marker);
write_text_element(&mut writer, "Prefix", &maybe_url_encode(prefix, encoding_type));
write_text_element(&mut writer, "Marker", &maybe_url_encode(marker, encoding_type));
write_text_element(&mut writer, "MaxKeys", &max_keys.to_string());
write_text_element(&mut writer, "IsTruncated", &is_truncated.to_string());
if !delimiter.is_empty() {
write_text_element(&mut writer, "Delimiter", delimiter);
write_text_element(
&mut writer,
"Delimiter",
&maybe_url_encode(delimiter, encoding_type),
);
}
if !delimiter.is_empty() && is_truncated {
if let Some(nm) = next_marker {
if !nm.is_empty() {
write_text_element(&mut writer, "NextMarker", nm);
write_text_element(&mut writer, "NextMarker", &maybe_url_encode(nm, encoding_type));
}
}
}
if let Some(encoding) = encoding_type {
if !encoding.is_empty() {
write_text_element(&mut writer, "EncodingType", encoding);
}
}
for obj in objects {
writer
.write_event(Event::Start(BytesStart::new("Contents")))
.unwrap();
write_text_element(&mut writer, "Key", &obj.key);
write_text_element(&mut writer, "Key", &maybe_url_encode(&obj.key, encoding_type));
write_text_element(
&mut writer,
"LastModified",
@@ -196,7 +300,7 @@ pub fn list_objects_v1_xml(
writer
.write_event(Event::Start(BytesStart::new("CommonPrefixes")))
.unwrap();
write_text_element(&mut writer, "Prefix", cp);
write_text_element(&mut writer, "Prefix", &maybe_url_encode(cp, encoding_type));
writer
.write_event(Event::End(BytesEnd::new("CommonPrefixes")))
.unwrap();
@@ -319,8 +423,15 @@ pub fn copy_object_result_xml(etag: &str, last_modified: &str) -> String {
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
pub struct DeletedEntry {
pub key: String,
pub version_id: Option<String>,
pub delete_marker: bool,
pub delete_marker_version_id: Option<String>,
}
pub fn delete_result_xml(
deleted: &[(String, Option<String>)],
deleted: &[DeletedEntry],
errors: &[(String, String, String)],
quiet: bool,
) -> String {
@@ -334,14 +445,20 @@ pub fn delete_result_xml(
writer.write_event(Event::Start(start)).unwrap();
if !quiet {
for (key, version_id) in deleted {
for entry in deleted {
writer
.write_event(Event::Start(BytesStart::new("Deleted")))
.unwrap();
write_text_element(&mut writer, "Key", key);
if let Some(vid) = version_id {
write_text_element(&mut writer, "Key", &entry.key);
if let Some(ref vid) = entry.version_id {
write_text_element(&mut writer, "VersionId", vid);
}
if entry.delete_marker {
write_text_element(&mut writer, "DeleteMarker", "true");
if let Some(ref dm_vid) = entry.delete_marker_version_id {
write_text_element(&mut writer, "DeleteMarkerVersionId", dm_vid);
}
}
writer
.write_event(Event::End(BytesEnd::new("Deleted")))
.unwrap();

Some files were not shown because too many files have changed in this diff Show More