kqjy/MyFSIO
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

Fix S3 versioning/delete markers, path-safety leaks, and error-code conformance; parallelize DeleteObjects; restore per-op rate limits

Browse Source
This commit is contained in:
kqjy
2026-04-23 20:23:11 +08:00
parent 7ef3820f6e
commit bd405cc2fe
19 changed files with 893 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
crates/myfsio-server/src/config.rs
View File

@@ -83,6 +83,10 @@ pub struct ServerConfig {
pub stream_chunk_size: usize,
pub request_body_timeout_secs: u64,
pub ratelimit_default: RateLimitSetting,
pub ratelimit_list_buckets: RateLimitSetting,
pub ratelimit_bucket_ops: RateLimitSetting,
pub ratelimit_object_ops: RateLimitSetting,
pub ratelimit_head_ops: RateLimitSetting,
pub ratelimit_admin: RateLimitSetting,
pub ratelimit_storage_uri: String,
pub ui_enabled: bool,
@@ -228,7 +232,15 @@ impl ServerConfig {
let stream_chunk_size = parse_usize_env("STREAM_CHUNK_SIZE", 1_048_576);
let request_body_timeout_secs = parse_u64_env("REQUEST_BODY_TIMEOUT_SECONDS", 60);
let ratelimit_default =
parse_rate_limit_env("RATE_LIMIT_DEFAULT", RateLimitSetting::new(200, 60));
parse_rate_limit_env("RATE_LIMIT_DEFAULT", RateLimitSetting::new(500, 60));
let ratelimit_list_buckets =
parse_rate_limit_env("RATE_LIMIT_LIST_BUCKETS", ratelimit_default);
let ratelimit_bucket_ops =
parse_rate_limit_env("RATE_LIMIT_BUCKET_OPS", ratelimit_default);
let ratelimit_object_ops =
parse_rate_limit_env("RATE_LIMIT_OBJECT_OPS", ratelimit_default);
let ratelimit_head_ops =
parse_rate_limit_env("RATE_LIMIT_HEAD_OPS", ratelimit_default);
let ratelimit_admin =
parse_rate_limit_env("RATE_LIMIT_ADMIN", RateLimitSetting::new(60, 60));
let ratelimit_storage_uri =
@@ -308,6 +320,10 @@ impl ServerConfig {
stream_chunk_size,
request_body_timeout_secs,
ratelimit_default,
ratelimit_list_buckets,
ratelimit_bucket_ops,
ratelimit_object_ops,
ratelimit_head_ops,
ratelimit_admin,
ratelimit_storage_uri,
ui_enabled,
@@ -391,7 +407,11 @@ impl Default for ServerConfig {
bulk_delete_max_keys: 1000,
stream_chunk_size: 1_048_576,
request_body_timeout_secs: 60,
ratelimit_default: RateLimitSetting::new(200, 60),
ratelimit_default: RateLimitSetting::new(500, 60),
ratelimit_list_buckets: RateLimitSetting::new(500, 60),
ratelimit_bucket_ops: RateLimitSetting::new(500, 60),
ratelimit_object_ops: RateLimitSetting::new(500, 60),
ratelimit_head_ops: RateLimitSetting::new(500, 60),
ratelimit_admin: RateLimitSetting::new(60, 60),
ratelimit_storage_uri: "memory://".to_string(),
ui_enabled: true,
@@ -476,7 +496,31 @@ fn parse_list_env(key: &str, default: &str) -> Vec<String> {
}
pub fn parse_rate_limit(value: &str) -> Option<RateLimitSetting> {
let parts = value.split_whitespace().collect::<Vec<_>>();
let trimmed = value.trim();
if let Some((requests, window)) = trimmed.split_once('/') {
let max_requests = requests.trim().parse::<u32>().ok()?;
if max_requests == 0 {
return None;
}
let window_str = window.trim().to_ascii_lowercase();
let window_seconds = if let Ok(n) = window_str.parse::<u64>() {
if n == 0 {
return None;
}
n
} else {
match window_str.as_str() {
"s" | "sec" | "second" | "seconds" => 1,
"m" | "min" | "minute" | "minutes" => 60,
"h" | "hr" | "hour" | "hours" => 3600,
"d" | "day" | "days" => 86_400,
_ => return None,
}
};
return Some(RateLimitSetting::new(max_requests, window_seconds));
}
let parts = trimmed.split_whitespace().collect::<Vec<_>>();
if parts.len() != 3 || !parts[1].eq_ignore_ascii_case("per") {
return None;
}
@@ -521,6 +565,15 @@ mod tests {
parse_rate_limit("3 per hours"),
Some(RateLimitSetting::new(3, 3600))
);
assert_eq!(
parse_rate_limit("50000/60"),
Some(RateLimitSetting::new(50000, 60))
);
assert_eq!(
parse_rate_limit("100/minute"),
Some(RateLimitSetting::new(100, 60))
);
assert_eq!(parse_rate_limit("0/60"), None);
assert_eq!(parse_rate_limit("0 per minute"), None);
assert_eq!(parse_rate_limit("bad"), None);
}
@@ -536,7 +589,7 @@ mod tests {
assert_eq!(config.object_key_max_length_bytes, 1024);
assert_eq!(config.object_tag_limit, 50);
assert_eq!(config.ratelimit_default, RateLimitSetting::new(200, 60));
assert_eq!(config.ratelimit_default, RateLimitSetting::new(500, 60));
std::env::remove_var("OBJECT_TAG_LIMIT");
std::env::remove_var("RATE_LIMIT_DEFAULT");

6
crates/myfsio-server/src/handlers/config.rs
View File

@@ -1118,9 +1118,13 @@ pub async fn list_object_versions(
}
for obj in objects.iter().take(current_count) {
let version_id = obj.version_id.clone().unwrap_or_else(|| "null".to_string());
xml.push_str("<Version>");
xml.push_str(&format!("<Key>{}</Key>", xml_escape(&obj.key)));
xml.push_str("<VersionId>null</VersionId>");
xml.push_str(&format!(
"<VersionId>{}</VersionId>",
xml_escape(&version_id)
));
xml.push_str("<IsLatest>true</IsLatest>");
xml.push_str(&format!(
"<LastModified>{}</LastModified>",

340
crates/myfsio-server/src/handlers/mod.rs
View File

@@ -51,10 +51,64 @@ fn storage_err_response(err: myfsio_storage::error::StorageError) -> Response {
if let Some(message) = crate::middleware::sha_body::sha256_mismatch_message(io_err) {
return bad_digest_response(message);
}
if let Some(response) = io_error_to_s3_response(io_err) {
return response;
}
}
if let myfsio_storage::error::StorageError::DeleteMarker {
bucket,
key,
version_id,
} = &err
{
let s3_err = S3Error::from_code(S3ErrorCode::NoSuchKey)
.with_resource(format!("/{}/{}", bucket, key))
.with_request_id(uuid::Uuid::new_v4().simple().to_string());
let status = StatusCode::from_u16(s3_err.http_status())
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
let mut resp_headers = HeaderMap::new();
resp_headers.insert("x-amz-delete-marker", "true".parse().unwrap());
if let Ok(vid) = version_id.parse() {
resp_headers.insert("x-amz-version-id", vid);
}
resp_headers.insert("content-type", "application/xml".parse().unwrap());
return (status, resp_headers, s3_err.to_xml()).into_response();
}
s3_error_response(S3Error::from(err))
}
fn io_error_to_s3_response(err: &std::io::Error) -> Option<Response> {
use std::io::ErrorKind;
let message = err.to_string();
let lower = message.to_ascii_lowercase();
let hit_collision = matches!(
err.kind(),
ErrorKind::NotADirectory
| ErrorKind::IsADirectory
| ErrorKind::AlreadyExists
| ErrorKind::DirectoryNotEmpty
) || lower.contains("not a directory")
|| lower.contains("is a directory")
|| lower.contains("file exists")
|| lower.contains("directory not empty");
let hit_name_too_long = matches!(err.kind(), ErrorKind::InvalidFilename)
|| lower.contains("file name too long");
if !hit_collision && !hit_name_too_long {
return None;
}
let code = if hit_name_too_long {
S3ErrorCode::InvalidKey
} else {
S3ErrorCode::InvalidRequest
};
let detail = if hit_name_too_long {
"Object key exceeds the filesystem's per-segment length limit"
} else {
"Object key collides with an existing object path on the storage backend"
};
Some(s3_error_response(S3Error::new(code, detail)))
}
fn trigger_replication(state: &AppState, bucket: &str, key: &str, action: &str) {
let manager = state.replication.clone();
let bucket = bucket.to_string();
@@ -242,6 +296,8 @@ pub struct BucketQuery {
pub continuation_token: Option<String>,
#[serde(rename = "start-after")]
pub start_after: Option<String>,
#[serde(rename = "encoding-type")]
pub encoding_type: Option<String>,
pub uploads: Option<String>,
pub delete: Option<String>,
pub versioning: Option<String>,
@@ -490,11 +546,12 @@ pub async fn get_bucket(
} else {
None
};
let encoding_type = query.encoding_type.as_deref();
let xml = if is_v2 {
let next_token = next_marker
.as_deref()
.map(|s| URL_SAFE.encode(s.as_bytes()));
myfsio_xml::response::list_objects_v2_xml(
myfsio_xml::response::list_objects_v2_xml_with_encoding(
&bucket,
&prefix,
&delimiter,
@@ -505,9 +562,10 @@ pub async fn get_bucket(
query.continuation_token.as_deref(),
next_token.as_deref(),
result.objects.len(),
encoding_type,
)
} else {
myfsio_xml::response::list_objects_v1_xml(
myfsio_xml::response::list_objects_v1_xml_with_encoding(
&bucket,
&prefix,
&marker,
@@ -517,6 +575,7 @@ pub async fn get_bucket(
&[],
result.is_truncated,
next_marker.as_deref(),
encoding_type,
)
};
(StatusCode::OK, [("content-type", "application/xml")], xml).into_response()
@@ -532,12 +591,13 @@ pub async fn get_bucket(
};
match state.storage.list_objects_shallow(&bucket, &params).await {
Ok(result) => {
let encoding_type = query.encoding_type.as_deref();
let xml = if is_v2 {
let next_token = result
.next_continuation_token
.as_deref()
.map(|s| URL_SAFE.encode(s.as_bytes()));
myfsio_xml::response::list_objects_v2_xml(
myfsio_xml::response::list_objects_v2_xml_with_encoding(
&bucket,
&params.prefix,
&delimiter,
@@ -548,9 +608,10 @@ pub async fn get_bucket(
query.continuation_token.as_deref(),
next_token.as_deref(),
result.objects.len() + result.common_prefixes.len(),
encoding_type,
)
} else {
myfsio_xml::response::list_objects_v1_xml(
myfsio_xml::response::list_objects_v1_xml_with_encoding(
&bucket,
&params.prefix,
&marker,
@@ -560,6 +621,7 @@ pub async fn get_bucket(
&result.common_prefixes,
result.is_truncated,
result.next_continuation_token.as_deref(),
encoding_type,
)
};
(StatusCode::OK, [("content-type", "application/xml")], xml).into_response()
@@ -955,6 +1017,47 @@ fn has_upload_checksum(headers: &HeaderMap) -> bool {
|| headers.contains_key("x-amz-checksum-crc32")
}
fn persist_additional_checksums(headers: &HeaderMap, metadata: &mut HashMap<String, String>) {
for algo in [
"sha256", "sha1", "crc32", "crc32c", "crc64nvme",
] {
let header_name = format!("x-amz-checksum-{}", algo);
if let Some(value) = headers.get(&header_name).and_then(|v| v.to_str().ok()) {
let trimmed = value.trim();
if !trimmed.is_empty() {
metadata.insert(format!("__checksum_{}__", algo), trimmed.to_string());
}
}
}
if let Some(value) = headers
.get("x-amz-sdk-checksum-algorithm")
.and_then(|v| v.to_str().ok())
{
let trimmed = value.trim().to_ascii_uppercase();
if !trimmed.is_empty() {
metadata.insert("__checksum_algorithm__".to_string(), trimmed);
}
}
}
fn apply_stored_checksum_headers(resp_headers: &mut HeaderMap, metadata: &HashMap<String, String>) {
for algo in [
"sha256", "sha1", "crc32", "crc32c", "crc64nvme",
] {
if let Some(value) = metadata.get(&format!("__checksum_{}__", algo)) {
if let Ok(parsed) = value.parse() {
resp_headers.insert(
axum::http::HeaderName::from_bytes(
format!("x-amz-checksum-{}", algo).as_bytes(),
)
.unwrap(),
parsed,
);
}
}
}
}
fn validate_upload_checksums(headers: &HeaderMap, data: &[u8]) -> Result<(), Response> {
if let Some(expected) = base64_header_bytes(headers, "content-md5")? {
if expected.len() != 16 || Md5::digest(data).as_slice() != expected.as_slice() {
@@ -984,7 +1087,7 @@ fn validate_upload_checksums(headers: &HeaderMap, data: &[u8]) -> Result<(), Res
Ok(())
}
async fn collect_upload_body(body: Body, aws_chunked: bool) -> Result<Vec<u8>, Response> {
async fn collect_upload_body(body: Body, aws_chunked: bool) -> Result<bytes::Bytes, Response> {
if aws_chunked {
let mut reader = chunked::decode_body(body);
let mut data = Vec::new();
@@ -994,12 +1097,12 @@ async fn collect_upload_body(body: Body, aws_chunked: bool) -> Result<Vec<u8>, R
"Failed to read aws-chunked request body",
))
})?;
return Ok(data);
return Ok(bytes::Bytes::from(data));
}
http_body_util::BodyExt::collect(body)
.await
.map(|collected| collected.to_bytes().to_vec())
.map(|collected| collected.to_bytes())
.map_err(|err| {
if let Some(message) = crate::middleware::sha_body::sha256_mismatch_message(&err) {
bad_digest_response(message)
@@ -1213,6 +1316,8 @@ pub async fn put_object(
}
}
persist_additional_checksums(&headers, &mut metadata);
let aws_chunked = is_aws_chunked(&headers);
let boxed: myfsio_storage::traits::AsyncReadStream = if has_upload_checksum(&headers) {
let data = match collect_upload_body(body, aws_chunked).await {
@@ -1227,8 +1332,7 @@ pub async fn put_object(
Box::pin(chunked::decode_body(body))
} else {
let stream = tokio_util::io::StreamReader::new(
http_body_util::BodyStream::new(body)
.map_ok(|frame| frame.into_data().unwrap_or_default())
body.into_data_stream()
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)),
);
Box::pin(stream)
@@ -1288,10 +1392,16 @@ pub async fn put_object(
resp_headers
.insert("etag", format!("\"{}\"", etag).parse().unwrap());
}
if let Some(ref vid) = meta.version_id {
if let Ok(value) = vid.parse() {
resp_headers.insert("x-amz-version-id", value);
}
}
resp_headers.insert(
"x-amz-server-side-encryption",
enc_ctx.algorithm.as_str().parse().unwrap(),
);
apply_stored_checksum_headers(&mut resp_headers, &enc_metadata);
notifications::emit_object_created(
&state,
&bucket,
@@ -1321,6 +1431,17 @@ pub async fn put_object(
if let Some(ref etag) = meta.etag {
resp_headers.insert("etag", format!("\"{}\"", etag).parse().unwrap());
}
if let Some(ref vid) = meta.version_id {
if let Ok(value) = vid.parse() {
resp_headers.insert("x-amz-version-id", value);
}
}
let stored = state
.storage
.get_object_metadata(&bucket, &key)
.await
.unwrap_or_default();
apply_stored_checksum_headers(&mut resp_headers, &stored);
notifications::emit_object_created(
&state,
&bucket,
@@ -1450,7 +1571,7 @@ pub async fn get_object(
}
};
let file_size = file.metadata().await.map(|m| m.len()).unwrap_or(0);
let stream = ReaderStream::new(file);
let stream = ReaderStream::with_capacity(file, 256 * 1024);
let body = Body::from_stream(stream);
let meta = head_meta.clone();
@@ -1481,10 +1602,15 @@ pub async fn get_object(
enc_info.algorithm.parse().unwrap(),
);
apply_stored_response_headers(&mut resp_headers, &all_meta);
apply_stored_checksum_headers(&mut resp_headers, &all_meta);
if let Some(ref requested_version) = query.version_id {
if let Ok(value) = requested_version.parse() {
resp_headers.insert("x-amz-version-id", value);
}
} else if let Some(vid) = all_meta.get("__version_id__") {
if let Ok(value) = vid.parse() {
resp_headers.insert("x-amz-version-id", value);
}
}
apply_user_metadata(&mut resp_headers, &meta.metadata);
@@ -1506,7 +1632,7 @@ pub async fn get_object(
match object_result {
Ok((meta, reader)) => {
let stream = ReaderStream::new(reader);
let stream = ReaderStream::with_capacity(reader, 256 * 1024);
let body = Body::from_stream(stream);
let mut headers = HeaderMap::new();
@@ -1525,10 +1651,15 @@ pub async fn get_object(
);
headers.insert("accept-ranges", "bytes".parse().unwrap());
apply_stored_response_headers(&mut headers, &all_meta);
apply_stored_checksum_headers(&mut headers, &all_meta);
if let Some(ref requested_version) = query.version_id {
if let Ok(value) = requested_version.parse() {
headers.insert("x-amz-version-id", value);
}
} else if let Some(ref vid) = meta.version_id {
if let Ok(value) = vid.parse() {
headers.insert("x-amz-version-id", value);
}
}
apply_user_metadata(&mut headers, &meta.metadata);
@@ -1596,10 +1727,15 @@ pub async fn delete_object(
.delete_object_version(&bucket, &key, version_id)
.await
{
Ok(()) => {
Ok(outcome) => {
let mut resp_headers = HeaderMap::new();
if let Ok(value) = version_id.parse() {
resp_headers.insert("x-amz-version-id", value);
if let Some(ref vid) = outcome.version_id {
if let Ok(value) = vid.parse() {
resp_headers.insert("x-amz-version-id", value);
}
}
if outcome.is_delete_marker {
resp_headers.insert("x-amz-delete-marker", "true".parse().unwrap());
}
notifications::emit_object_removed(&state, &bucket, &key, "", "", "", "Delete");
trigger_replication(&state, &bucket, &key, "delete");
@@ -1616,10 +1752,19 @@ pub async fn delete_object(
}
match state.storage.delete_object(&bucket, &key).await {
Ok(()) => {
Ok(outcome) => {
let mut resp_headers = HeaderMap::new();
if let Some(ref vid) = outcome.version_id {
if let Ok(value) = vid.parse() {
resp_headers.insert("x-amz-version-id", value);
}
}
if outcome.is_delete_marker {
resp_headers.insert("x-amz-delete-marker", "true".parse().unwrap());
}
notifications::emit_object_removed(&state, &bucket, &key, "", "", "", "Delete");
trigger_replication(&state, &bucket, &key, "delete");
(StatusCode::NO_CONTENT, HeaderMap::new()).into_response()
(StatusCode::NO_CONTENT, resp_headers).into_response()
}
Err(e) => storage_err_response(e),
}
@@ -1678,10 +1823,15 @@ pub async fn head_object(
);
headers.insert("accept-ranges", "bytes".parse().unwrap());
apply_stored_response_headers(&mut headers, &all_meta);
apply_stored_checksum_headers(&mut headers, &all_meta);
if let Some(ref requested_version) = query.version_id {
if let Ok(value) = requested_version.parse() {
headers.insert("x-amz-version-id", value);
}
} else if let Some(ref vid) = meta.version_id {
if let Ok(value) = vid.parse() {
headers.insert("x-amz-version-id", value);
}
}
apply_user_metadata(&mut headers, &meta.metadata);
@@ -1714,8 +1864,7 @@ async fn upload_part_handler_with_chunking(
Box::pin(chunked::decode_body(body))
} else {
let stream = tokio_util::io::StreamReader::new(
http_body_util::BodyStream::new(body)
.map_ok(|frame| frame.into_data().unwrap_or_default())
body.into_data_stream()
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)),
);
Box::pin(stream)
@@ -2240,61 +2389,110 @@ async fn delete_objects_handler(state: &AppState, bucket: &str, body: Body) -> R
));
}
let mut deleted = Vec::new();
let mut errors = Vec::new();
use futures::stream::{self, StreamExt};
for obj in &parsed.objects {
if let Err(message) = match obj.version_id.as_deref() {
Some(version_id) if version_id != "null" => match state
.storage
.get_object_version_metadata(bucket, &obj.key, version_id)
.await
{
Ok(metadata) => object_lock::can_delete_object(&metadata, false),
Err(err) => Err(S3Error::from(err).message),
},
_ => match state.storage.head_object(bucket, &obj.key).await {
Ok(_) => match state.storage.get_object_metadata(bucket, &obj.key).await {
Ok(metadata) => object_lock::can_delete_object(&metadata, false),
Err(err) => Err(S3Error::from(err).message),
},
Err(myfsio_storage::error::StorageError::ObjectNotFound { .. }) => Ok(()),
Err(err) => Err(S3Error::from(err).message),
},
} {
errors.push((
obj.key.clone(),
S3ErrorCode::AccessDenied.as_str().to_string(),
message,
));
continue;
}
let delete_result = if let Some(version_id) = obj.version_id.as_deref() {
if version_id == "null" {
state.storage.delete_object(bucket, &obj.key).await
} else {
state
.storage
.delete_object_version(bucket, &obj.key, version_id)
.await
}
} else {
state.storage.delete_object(bucket, &obj.key).await
};
let results: Vec<(String, Option<String>, Result<myfsio_common::types::DeleteOutcome, (String, String)>)> =
stream::iter(parsed.objects.iter().cloned())
.map(|obj| {
let state = state.clone();
let bucket = bucket.to_string();
async move {
let key = obj.key.clone();
let requested_vid = obj.version_id.clone();
let lock_check: Result<(), (String, String)> = match obj.version_id.as_deref() {
Some(version_id) if version_id != "null" => match state
.storage
.get_object_version_metadata(&bucket, &obj.key, version_id)
.await
{
Ok(metadata) => object_lock::can_delete_object(&metadata, false)
.map_err(|m| {
(S3ErrorCode::AccessDenied.as_str().to_string(), m)
}),
Err(err) => {
let s3err = S3Error::from(err);
Err((s3err.code.as_str().to_string(), s3err.message))
}
},
_ => match state.storage.head_object(&bucket, &obj.key).await {
Ok(_) => {
match state
.storage
.get_object_metadata(&bucket, &obj.key)
.await
{
Ok(metadata) => object_lock::can_delete_object(&metadata, false)
.map_err(|m| {
(
S3ErrorCode::AccessDenied.as_str().to_string(),
m,
)
}),
Err(err) => {
let s3err = S3Error::from(err);
Err((s3err.code.as_str().to_string(), s3err.message))
}
}
}
Err(myfsio_storage::error::StorageError::ObjectNotFound { .. }) => {
Ok(())
}
Err(myfsio_storage::error::StorageError::DeleteMarker { .. }) => {
Ok(())
}
Err(err) => {
let s3err = S3Error::from(err);
Err((s3err.code.as_str().to_string(), s3err.message))
}
},
};
match delete_result {
Ok(()) => {
notifications::emit_object_removed(state, bucket, &obj.key, "", "", "", "Delete");
trigger_replication(state, bucket, &obj.key, "delete");
deleted.push((obj.key.clone(), obj.version_id.clone()))
let result = match lock_check {
Err(e) => Err(e),
Ok(()) => {
let outcome = match obj.version_id.as_deref() {
Some(version_id) if version_id != "null" => {
state
.storage
.delete_object_version(&bucket, &obj.key, version_id)
.await
}
_ => state.storage.delete_object(&bucket, &obj.key).await,
};
outcome.map_err(|e| {
let s3err = S3Error::from(e);
(s3err.code.as_str().to_string(), s3err.message)
})
}
};
(key, requested_vid, result)
}
})
.buffer_unordered(32)
.collect()
.await;
let mut deleted: Vec<myfsio_xml::response::DeletedEntry> = Vec::new();
let mut errors: Vec<(String, String, String)> = Vec::new();
for (key, requested_vid, result) in results {
match result {
Ok(outcome) => {
notifications::emit_object_removed(state, bucket, &key, "", "", "", "Delete");
trigger_replication(state, bucket, &key, "delete");
let delete_marker_version_id = if outcome.is_delete_marker {
outcome.version_id.clone()
} else {
None
};
deleted.push(myfsio_xml::response::DeletedEntry {
key,
version_id: requested_vid,
delete_marker: outcome.is_delete_marker,
delete_marker_version_id,
});
}
Err(e) => {
let s3err = S3Error::from(e);
errors.push((
obj.key.clone(),
s3err.code.as_str().to_string(),
s3err.message,
));
Err((code, message)) => {
errors.push((key, code, message));
}
}
}
@@ -2366,7 +2564,7 @@ async fn range_get_handler(
let length = end - start + 1;
let limited = file.take(length);
let stream = ReaderStream::new(limited);
let stream = ReaderStream::with_capacity(limited, 256 * 1024);
let body = Body::from_stream(stream);
let mut headers = HeaderMap::new();

8
crates/myfsio-server/src/handlers/ui_api.rs
View File

@@ -121,6 +121,8 @@ fn storage_status(err: &StorageError) -> StatusCode {
| StorageError::ObjectNotFound { .. }
| StorageError::VersionNotFound { .. }
| StorageError::UploadNotFound(_) => StatusCode::NOT_FOUND,
StorageError::DeleteMarker { .. } => StatusCode::NOT_FOUND,
StorageError::MethodNotAllowed(_) => StatusCode::METHOD_NOT_ALLOWED,
StorageError::InvalidBucketName(_)
| StorageError::InvalidObjectKey(_)
| StorageError::InvalidRange
@@ -2599,7 +2601,7 @@ async fn move_object_json(state: &AppState, bucket: &str, key: &str, body: Body)
match state.storage.copy_object(bucket, key, dest_bucket, dest_key).await {
Ok(_) => match state.storage.delete_object(bucket, key).await {
Ok(()) => {
Ok(_) => {
super::trigger_replication(state, dest_bucket, dest_key, "write");
super::trigger_replication(state, bucket, key, "delete");
Json(json!({
@@ -2674,7 +2676,7 @@ async fn delete_object_json(
}
match state.storage.delete_object(bucket, key).await {
Ok(()) => {
Ok(_) => {
super::trigger_replication(state, bucket, key, "delete");
Json(json!({
"status": "ok",
@@ -2953,7 +2955,7 @@ pub async fn bulk_delete_objects(
for key in keys {
match state.storage.delete_object(&bucket_name, &key).await {
Ok(()) => {
Ok(_) => {
super::trigger_replication(&state, &bucket_name, &key, "delete");
if payload.purge_versions {
if let Err(err) =

6
crates/myfsio-server/src/lib.rs
View File

@@ -335,8 +335,12 @@ pub fn create_ui_router(state: state::AppState) -> Router {
}
pub fn create_router(state: state::AppState) -> Router {
let default_rate_limit = middleware::RateLimitLayerState::new(
let default_rate_limit = middleware::RateLimitLayerState::with_per_op(
state.config.ratelimit_default,
state.config.ratelimit_list_buckets,
state.config.ratelimit_bucket_ops,
state.config.ratelimit_object_ops,
state.config.ratelimit_head_ops,
state.config.num_trusted_proxies,
);
let admin_rate_limit = middleware::RateLimitLayerState::new(

9
crates/myfsio-server/src/middleware/auth.rs
View File

@@ -1344,7 +1344,7 @@ fn verify_sigv4_query(state: &AppState, req: &Request) -> AuthResult {
}
if elapsed < -(state.config.sigv4_timestamp_tolerance_secs as i64) {
return AuthResult::Denied(S3Error::new(
S3ErrorCode::AccessDenied,
S3ErrorCode::RequestTimeTooSkewed,
"Request is too far in the future",
));
}
@@ -1414,8 +1414,11 @@ fn check_timestamp_freshness(amz_date: &str, tolerance_secs: u64) -> Option<S3Er
if diff > tolerance_secs {
return Some(S3Error::new(
S3ErrorCode::AccessDenied,
"Request timestamp too old or too far in the future",
S3ErrorCode::RequestTimeTooSkewed,
format!(
"The difference between the request time and the server's time is too large ({}s, tolerance {}s)",
diff, tolerance_secs
),
));
}
None

69
crates/myfsio-server/src/middleware/ratelimit.rs
View File

@@ -4,7 +4,7 @@ use std::sync::Arc;
use std::time::{Duration, Instant};
use axum::extract::{ConnectInfo, Request, State};
use axum::http::{header, StatusCode};
use axum::http::{header, Method, StatusCode};
use axum::middleware::Next;
use axum::response::{IntoResponse, Response};
use parking_lot::Mutex;
@@ -13,17 +13,77 @@ use crate::config::RateLimitSetting;
#[derive(Clone)]
pub struct RateLimitLayerState {
limiter: Arc<FixedWindowLimiter>,
default_limiter: Arc<FixedWindowLimiter>,
list_buckets_limiter: Option<Arc<FixedWindowLimiter>>,
bucket_ops_limiter: Option<Arc<FixedWindowLimiter>>,
object_ops_limiter: Option<Arc<FixedWindowLimiter>>,
head_ops_limiter: Option<Arc<FixedWindowLimiter>>,
num_trusted_proxies: usize,
}
impl RateLimitLayerState {
pub fn new(setting: RateLimitSetting, num_trusted_proxies: usize) -> Self {
Self {
limiter: Arc::new(FixedWindowLimiter::new(setting)),
default_limiter: Arc::new(FixedWindowLimiter::new(setting)),
list_buckets_limiter: None,
bucket_ops_limiter: None,
object_ops_limiter: None,
head_ops_limiter: None,
num_trusted_proxies,
}
}
pub fn with_per_op(
default: RateLimitSetting,
list_buckets: RateLimitSetting,
bucket_ops: RateLimitSetting,
object_ops: RateLimitSetting,
head_ops: RateLimitSetting,
num_trusted_proxies: usize,
) -> Self {
Self {
default_limiter: Arc::new(FixedWindowLimiter::new(default)),
list_buckets_limiter: (list_buckets != default)
.then(|| Arc::new(FixedWindowLimiter::new(list_buckets))),
bucket_ops_limiter: (bucket_ops != default)
.then(|| Arc::new(FixedWindowLimiter::new(bucket_ops))),
object_ops_limiter: (object_ops != default)
.then(|| Arc::new(FixedWindowLimiter::new(object_ops))),
head_ops_limiter: (head_ops != default)
.then(|| Arc::new(FixedWindowLimiter::new(head_ops))),
num_trusted_proxies,
}
}
fn select_limiter(&self, req: &Request) -> &Arc<FixedWindowLimiter> {
let path = req.uri().path();
let method = req.method();
if path == "/" && *method == Method::GET {
if let Some(ref limiter) = self.list_buckets_limiter {
return limiter;
}
}
let segments: Vec<&str> = path
.trim_start_matches('/')
.split('/')
.filter(|s| !s.is_empty())
.collect();
if *method == Method::HEAD {
if let Some(ref limiter) = self.head_ops_limiter {
return limiter;
}
}
if segments.len() == 1 {
if let Some(ref limiter) = self.bucket_ops_limiter {
return limiter;
}
} else if segments.len() >= 2 {
if let Some(ref limiter) = self.object_ops_limiter {
return limiter;
}
}
&self.default_limiter
}
}
#[derive(Debug)]
@@ -99,7 +159,8 @@ pub async fn rate_limit_layer(
next: Next,
) -> Response {
let key = rate_limit_key(&req, state.num_trusted_proxies);
match state.limiter.check(&key) {
let limiter = state.select_limiter(&req);
match limiter.check(&key) {
Ok(()) => next.run(req).await,
Err(retry_after) => too_many_requests(retry_after),
}

40
crates/myfsio-server/tests/integration.rs
View File

@@ -121,6 +121,10 @@ fn test_app_with_rate_limits(
storage_root: tmp.path().to_path_buf(),
iam_config_path: iam_path.join("iam.json"),
ratelimit_default: default,
ratelimit_list_buckets: default,
ratelimit_bucket_ops: default,
ratelimit_object_ops: default,
ratelimit_head_ops: default,
ratelimit_admin: admin,
ui_enabled: false,
..myfsio_server::config::ServerConfig::default()
@@ -2398,7 +2402,7 @@ async fn test_versioned_object_can_be_read_and_deleted_by_version_id() {
}
#[tokio::test]
async fn test_versioned_put_and_delete_do_not_advertise_unstored_ids() {
async fn test_versioned_put_and_delete_emit_version_headers_and_delete_markers() {
let (app, _tmp) = test_app();
app.clone()
@@ -2430,7 +2434,14 @@ async fn test_versioned_put_and_delete_do_not_advertise_unstored_ids() {
.await
.unwrap();
assert_eq!(put_resp.status(), StatusCode::OK);
assert!(!put_resp.headers().contains_key("x-amz-version-id"));
let first_version = put_resp
.headers()
.get("x-amz-version-id")
.expect("PUT on versioned bucket must emit x-amz-version-id")
.to_str()
.unwrap()
.to_string();
assert!(!first_version.is_empty());
let overwrite_resp = app
.clone()
@@ -2442,7 +2453,14 @@ async fn test_versioned_put_and_delete_do_not_advertise_unstored_ids() {
.await
.unwrap();
assert_eq!(overwrite_resp.status(), StatusCode::OK);
assert!(!overwrite_resp.headers().contains_key("x-amz-version-id"));
let second_version = overwrite_resp
.headers()
.get("x-amz-version-id")
.expect("overwrite on versioned bucket must emit a new x-amz-version-id")
.to_str()
.unwrap()
.to_string();
assert_ne!(first_version, second_version);
let delete_resp = app
.clone()
@@ -2454,8 +2472,14 @@ async fn test_versioned_put_and_delete_do_not_advertise_unstored_ids() {
.await
.unwrap();
assert_eq!(delete_resp.status(), StatusCode::NO_CONTENT);
assert!(!delete_resp.headers().contains_key("x-amz-version-id"));
assert!(!delete_resp.headers().contains_key("x-amz-delete-marker"));
assert_eq!(
delete_resp
.headers()
.get("x-amz-delete-marker")
.and_then(|v| v.to_str().ok()),
Some("true")
);
assert!(delete_resp.headers().contains_key("x-amz-version-id"));
let versions_resp = app
.oneshot(signed_request(
@@ -2475,7 +2499,11 @@ async fn test_versioned_put_and_delete_do_not_advertise_unstored_ids() {
.to_vec(),
)
.unwrap();
assert!(!versions_body.contains("<DeleteMarker>"));
assert!(
versions_body.contains("<DeleteMarker>"),
"expected DeleteMarker entry in ListObjectVersions output, got: {}",
versions_body
);
}
#[tokio::test]