Applied max-keys to combined current + archived ListObjectVersions output and reports truncation
This commit is contained in:
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
MyFSIO is an S3-compatible object storage server with a Rust runtime and a filesystem-backed storage engine. The active server lives under `rust/myfsio-engine` and serves both the S3 API and the built-in web UI from a single process.
|
MyFSIO is an S3-compatible object storage server with a Rust runtime and a filesystem-backed storage engine. The active server lives under `rust/myfsio-engine` and serves both the S3 API and the built-in web UI from a single process.
|
||||||
|
|
||||||
The repository still contains a `python/` tree, but you do not need Python to run the current server.
|
The `python/` implementation is deprecated as of 2026-04-21. It remains in the repository for migration reference and legacy tests, but new development and supported runtime usage should target the Rust server.
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
|
|||||||
4
docs.md
4
docs.md
@@ -2,6 +2,8 @@
|
|||||||
|
|
||||||
This document describes the current Rust server in `rust/myfsio-engine`. It replaces the older Python-oriented runbook.
|
This document describes the current Rust server in `rust/myfsio-engine`. It replaces the older Python-oriented runbook.
|
||||||
|
|
||||||
|
The `python/` implementation is deprecated as of 2026-04-21. It is retained for migration reference and legacy validation only; production usage and new development should use the Rust server.
|
||||||
|
|
||||||
## 1. What Changed
|
## 1. What Changed
|
||||||
|
|
||||||
The active runtime is now Rust:
|
The active runtime is now Rust:
|
||||||
@@ -11,7 +13,7 @@ The active runtime is now Rust:
|
|||||||
- The main development workflow is `cargo run -p myfsio-server --`.
|
- The main development workflow is `cargo run -p myfsio-server --`.
|
||||||
- API-only mode is controlled with `UI_ENABLED=false`.
|
- API-only mode is controlled with `UI_ENABLED=false`.
|
||||||
|
|
||||||
The `python/` directory may still contain older implementation code, templates, and tests, but it is not required to run the current server.
|
The deprecated `python/` directory may still contain older implementation code, templates, and tests, but it is not required to run the current server.
|
||||||
|
|
||||||
## 2. Quick Start
|
## 2. Quick Start
|
||||||
|
|
||||||
|
|||||||
14
python/README.md
Normal file
14
python/README.md
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
# Deprecated Python Implementation
|
||||||
|
|
||||||
|
The Python implementation of MyFSIO is deprecated as of 2026-04-21.
|
||||||
|
|
||||||
|
The supported server runtime now lives in `../rust/myfsio-engine` and serves the S3 API and web UI from the Rust `myfsio-server` binary. Keep this tree for migration reference, compatibility checks, and legacy tests only.
|
||||||
|
|
||||||
|
For normal development and operations, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd ../rust/myfsio-engine
|
||||||
|
cargo run -p myfsio-server --
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not add new product features to the Python implementation unless they are needed to unblock a migration or compare behavior with the Rust server.
|
||||||
@@ -720,7 +720,7 @@ def _configure_logging(app: Flask) -> None:
|
|||||||
|
|
||||||
def _website_error_response(status_code, message):
|
def _website_error_response(status_code, message):
|
||||||
if status_code == 404:
|
if status_code == 404:
|
||||||
body = "404 page not found"
|
body = "<h1>404 page not found</h1>"
|
||||||
else:
|
else:
|
||||||
body = f"{status_code} {message}"
|
body = f"{status_code} {message}"
|
||||||
return Response(body, status=status_code, mimetype="text/html")
|
return Response(body, status=status_code, mimetype="text/html")
|
||||||
|
|||||||
@@ -28,6 +28,11 @@ from app.config import AppConfig
|
|||||||
from app.iam import IamService, IamError, ALLOWED_ACTIONS, _derive_fernet_key
|
from app.iam import IamService, IamError, ALLOWED_ACTIONS, _derive_fernet_key
|
||||||
from app.version import get_version
|
from app.version import get_version
|
||||||
|
|
||||||
|
PYTHON_DEPRECATION_MESSAGE = (
|
||||||
|
"The Python MyFSIO runtime is deprecated as of 2026-04-21. "
|
||||||
|
"Use the Rust server in rust/myfsio-engine for supported development and production usage."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _server_host() -> str:
|
def _server_host() -> str:
|
||||||
"""Return the bind host for API and UI servers."""
|
"""Return the bind host for API and UI servers."""
|
||||||
@@ -233,6 +238,8 @@ if __name__ == "__main__":
|
|||||||
parser.add_argument("--version", action="version", version=f"MyFSIO {get_version()}")
|
parser.add_argument("--version", action="version", version=f"MyFSIO {get_version()}")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
warnings.warn(PYTHON_DEPRECATION_MESSAGE, DeprecationWarning, stacklevel=1)
|
||||||
|
|
||||||
if args.reset_cred or args.mode == "reset-cred":
|
if args.reset_cred or args.mode == "reset-cred":
|
||||||
reset_credentials()
|
reset_credentials()
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|||||||
@@ -439,4 +439,4 @@ class TestWebsiteServing:
|
|||||||
store.set_mapping("noerr.example.com", "no-err")
|
store.set_mapping("noerr.example.com", "no-err")
|
||||||
resp = website_client.get("/missing.html", headers={"Host": "noerr.example.com"})
|
resp = website_client.get("/missing.html", headers={"Host": "noerr.example.com"})
|
||||||
assert resp.status_code == 404
|
assert resp.status_code == 404
|
||||||
assert resp.data == b"404 page not found"
|
assert resp.data == b"<h1>404 page not found</h1>"
|
||||||
|
|||||||
2
rust/myfsio-engine/Cargo.lock
generated
2
rust/myfsio-engine/Cargo.lock
generated
@@ -2707,6 +2707,7 @@ dependencies = [
|
|||||||
"futures",
|
"futures",
|
||||||
"http-body-util",
|
"http-body-util",
|
||||||
"hyper 1.9.0",
|
"hyper 1.9.0",
|
||||||
|
"md-5 0.10.6",
|
||||||
"mime_guess",
|
"mime_guess",
|
||||||
"multer",
|
"multer",
|
||||||
"myfsio-auth",
|
"myfsio-auth",
|
||||||
@@ -2723,6 +2724,7 @@ dependencies = [
|
|||||||
"roxmltree",
|
"roxmltree",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"sha2 0.10.9",
|
||||||
"subtle",
|
"subtle",
|
||||||
"sysinfo",
|
"sysinfo",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ use std::fmt;
|
|||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub enum S3ErrorCode {
|
pub enum S3ErrorCode {
|
||||||
AccessDenied,
|
AccessDenied,
|
||||||
|
BadDigest,
|
||||||
BucketAlreadyExists,
|
BucketAlreadyExists,
|
||||||
BucketNotEmpty,
|
BucketNotEmpty,
|
||||||
EntityTooLarge,
|
EntityTooLarge,
|
||||||
@@ -14,6 +15,7 @@ pub enum S3ErrorCode {
|
|||||||
InvalidPolicyDocument,
|
InvalidPolicyDocument,
|
||||||
InvalidRange,
|
InvalidRange,
|
||||||
InvalidRequest,
|
InvalidRequest,
|
||||||
|
InvalidTag,
|
||||||
MalformedXML,
|
MalformedXML,
|
||||||
MethodNotAllowed,
|
MethodNotAllowed,
|
||||||
NoSuchBucket,
|
NoSuchBucket,
|
||||||
@@ -32,6 +34,7 @@ impl S3ErrorCode {
|
|||||||
pub fn http_status(&self) -> u16 {
|
pub fn http_status(&self) -> u16 {
|
||||||
match self {
|
match self {
|
||||||
Self::AccessDenied => 403,
|
Self::AccessDenied => 403,
|
||||||
|
Self::BadDigest => 400,
|
||||||
Self::BucketAlreadyExists => 409,
|
Self::BucketAlreadyExists => 409,
|
||||||
Self::BucketNotEmpty => 409,
|
Self::BucketNotEmpty => 409,
|
||||||
Self::EntityTooLarge => 413,
|
Self::EntityTooLarge => 413,
|
||||||
@@ -43,6 +46,7 @@ impl S3ErrorCode {
|
|||||||
Self::InvalidPolicyDocument => 400,
|
Self::InvalidPolicyDocument => 400,
|
||||||
Self::InvalidRange => 416,
|
Self::InvalidRange => 416,
|
||||||
Self::InvalidRequest => 400,
|
Self::InvalidRequest => 400,
|
||||||
|
Self::InvalidTag => 400,
|
||||||
Self::MalformedXML => 400,
|
Self::MalformedXML => 400,
|
||||||
Self::MethodNotAllowed => 405,
|
Self::MethodNotAllowed => 405,
|
||||||
Self::NoSuchBucket => 404,
|
Self::NoSuchBucket => 404,
|
||||||
@@ -61,6 +65,7 @@ impl S3ErrorCode {
|
|||||||
pub fn as_str(&self) -> &'static str {
|
pub fn as_str(&self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
Self::AccessDenied => "AccessDenied",
|
Self::AccessDenied => "AccessDenied",
|
||||||
|
Self::BadDigest => "BadDigest",
|
||||||
Self::BucketAlreadyExists => "BucketAlreadyExists",
|
Self::BucketAlreadyExists => "BucketAlreadyExists",
|
||||||
Self::BucketNotEmpty => "BucketNotEmpty",
|
Self::BucketNotEmpty => "BucketNotEmpty",
|
||||||
Self::EntityTooLarge => "EntityTooLarge",
|
Self::EntityTooLarge => "EntityTooLarge",
|
||||||
@@ -72,6 +77,7 @@ impl S3ErrorCode {
|
|||||||
Self::InvalidPolicyDocument => "InvalidPolicyDocument",
|
Self::InvalidPolicyDocument => "InvalidPolicyDocument",
|
||||||
Self::InvalidRange => "InvalidRange",
|
Self::InvalidRange => "InvalidRange",
|
||||||
Self::InvalidRequest => "InvalidRequest",
|
Self::InvalidRequest => "InvalidRequest",
|
||||||
|
Self::InvalidTag => "InvalidTag",
|
||||||
Self::MalformedXML => "MalformedXML",
|
Self::MalformedXML => "MalformedXML",
|
||||||
Self::MethodNotAllowed => "MethodNotAllowed",
|
Self::MethodNotAllowed => "MethodNotAllowed",
|
||||||
Self::NoSuchBucket => "NoSuchBucket",
|
Self::NoSuchBucket => "NoSuchBucket",
|
||||||
@@ -90,6 +96,7 @@ impl S3ErrorCode {
|
|||||||
pub fn default_message(&self) -> &'static str {
|
pub fn default_message(&self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
Self::AccessDenied => "Access Denied",
|
Self::AccessDenied => "Access Denied",
|
||||||
|
Self::BadDigest => "The Content-MD5 or checksum value you specified did not match what we received",
|
||||||
Self::BucketAlreadyExists => "The requested bucket name is not available",
|
Self::BucketAlreadyExists => "The requested bucket name is not available",
|
||||||
Self::BucketNotEmpty => "The bucket you tried to delete is not empty",
|
Self::BucketNotEmpty => "The bucket you tried to delete is not empty",
|
||||||
Self::EntityTooLarge => "Your proposed upload exceeds the maximum allowed size",
|
Self::EntityTooLarge => "Your proposed upload exceeds the maximum allowed size",
|
||||||
@@ -101,6 +108,7 @@ impl S3ErrorCode {
|
|||||||
Self::InvalidPolicyDocument => "The content of the form does not meet the conditions specified in the policy document",
|
Self::InvalidPolicyDocument => "The content of the form does not meet the conditions specified in the policy document",
|
||||||
Self::InvalidRange => "The requested range is not satisfiable",
|
Self::InvalidRange => "The requested range is not satisfiable",
|
||||||
Self::InvalidRequest => "Invalid request",
|
Self::InvalidRequest => "Invalid request",
|
||||||
|
Self::InvalidTag => "The Tagging header is invalid",
|
||||||
Self::MalformedXML => "The XML you provided was not well-formed",
|
Self::MalformedXML => "The XML you provided was not well-formed",
|
||||||
Self::MethodNotAllowed => "The specified method is not allowed against this resource",
|
Self::MethodNotAllowed => "The specified method is not allowed against this resource",
|
||||||
Self::NoSuchBucket => "The specified bucket does not exist",
|
Self::NoSuchBucket => "The specified bucket does not exist",
|
||||||
|
|||||||
@@ -112,6 +112,8 @@ pub struct VersionInfo {
|
|||||||
pub last_modified: DateTime<Utc>,
|
pub last_modified: DateTime<Utc>,
|
||||||
pub etag: Option<String>,
|
pub etag: Option<String>,
|
||||||
pub is_latest: bool,
|
pub is_latest: bool,
|
||||||
|
#[serde(default)]
|
||||||
|
pub is_delete_marker: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ myfsio-crypto = { path = "../myfsio-crypto" }
|
|||||||
myfsio-storage = { path = "../myfsio-storage" }
|
myfsio-storage = { path = "../myfsio-storage" }
|
||||||
myfsio-xml = { path = "../myfsio-xml" }
|
myfsio-xml = { path = "../myfsio-xml" }
|
||||||
base64 = { workspace = true }
|
base64 = { workspace = true }
|
||||||
|
md-5 = { workspace = true }
|
||||||
axum = { workspace = true }
|
axum = { workspace = true }
|
||||||
tokio = { workspace = true }
|
tokio = { workspace = true }
|
||||||
tower = { workspace = true }
|
tower = { workspace = true }
|
||||||
@@ -29,6 +30,7 @@ percent-encoding = { workspace = true }
|
|||||||
quick-xml = { workspace = true }
|
quick-xml = { workspace = true }
|
||||||
mime_guess = "2"
|
mime_guess = "2"
|
||||||
crc32fast = { workspace = true }
|
crc32fast = { workspace = true }
|
||||||
|
sha2 = { workspace = true }
|
||||||
duckdb = { workspace = true }
|
duckdb = { workspace = true }
|
||||||
roxmltree = "0.20"
|
roxmltree = "0.20"
|
||||||
parking_lot = { workspace = true }
|
parking_lot = { workspace = true }
|
||||||
|
|||||||
@@ -1038,7 +1038,12 @@ fn s3_error_response(code: S3ErrorCode, message: &str, status: StatusCode) -> Re
|
|||||||
(status, [("content-type", "application/xml")], err.to_xml()).into_response()
|
(status, [("content-type", "application/xml")], err.to_xml()).into_response()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn list_object_versions(state: &AppState, bucket: &str) -> Response {
|
pub async fn list_object_versions(
|
||||||
|
state: &AppState,
|
||||||
|
bucket: &str,
|
||||||
|
prefix: Option<&str>,
|
||||||
|
max_keys: usize,
|
||||||
|
) -> Response {
|
||||||
match state.storage.list_buckets().await {
|
match state.storage.list_buckets().await {
|
||||||
Ok(buckets) => {
|
Ok(buckets) => {
|
||||||
if !buckets.iter().any(|b| b.name == bucket) {
|
if !buckets.iter().any(|b| b.name == bucket) {
|
||||||
@@ -1050,13 +1055,24 @@ pub async fn list_object_versions(state: &AppState, bucket: &str) -> Response {
|
|||||||
Err(e) => return storage_err(e),
|
Err(e) => return storage_err(e),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let fetch_limit = max_keys.saturating_add(1).max(1);
|
||||||
let params = myfsio_common::types::ListParams {
|
let params = myfsio_common::types::ListParams {
|
||||||
max_keys: 1000,
|
max_keys: fetch_limit,
|
||||||
|
prefix: prefix.map(ToOwned::to_owned),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
let objects = match state.storage.list_objects(bucket, ¶ms).await {
|
let object_result = match state.storage.list_objects(bucket, ¶ms).await {
|
||||||
Ok(result) => result.objects,
|
Ok(result) => result,
|
||||||
|
Err(e) => return storage_err(e),
|
||||||
|
};
|
||||||
|
let objects = object_result.objects;
|
||||||
|
let archived_versions = match state
|
||||||
|
.storage
|
||||||
|
.list_bucket_object_versions(bucket, prefix)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(versions) => versions,
|
||||||
Err(e) => return storage_err(e),
|
Err(e) => return storage_err(e),
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1064,11 +1080,24 @@ pub async fn list_object_versions(state: &AppState, bucket: &str) -> Response {
|
|||||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||||
<ListVersionsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">",
|
<ListVersionsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">",
|
||||||
);
|
);
|
||||||
xml.push_str(&format!("<Name>{}</Name>", bucket));
|
xml.push_str(&format!("<Name>{}</Name>", xml_escape(bucket)));
|
||||||
|
xml.push_str(&format!(
|
||||||
|
"<Prefix>{}</Prefix>",
|
||||||
|
xml_escape(prefix.unwrap_or(""))
|
||||||
|
));
|
||||||
|
xml.push_str(&format!("<MaxKeys>{}</MaxKeys>", max_keys));
|
||||||
|
|
||||||
for obj in &objects {
|
let current_count = objects.len().min(max_keys);
|
||||||
|
let remaining = max_keys.saturating_sub(current_count);
|
||||||
|
let archived_count = archived_versions.len().min(remaining);
|
||||||
|
let is_truncated = object_result.is_truncated
|
||||||
|
|| objects.len() > current_count
|
||||||
|
|| archived_versions.len() > archived_count;
|
||||||
|
xml.push_str(&format!("<IsTruncated>{}</IsTruncated>", is_truncated));
|
||||||
|
|
||||||
|
for obj in objects.iter().take(current_count) {
|
||||||
xml.push_str("<Version>");
|
xml.push_str("<Version>");
|
||||||
xml.push_str(&format!("<Key>{}</Key>", obj.key));
|
xml.push_str(&format!("<Key>{}</Key>", xml_escape(&obj.key)));
|
||||||
xml.push_str("<VersionId>null</VersionId>");
|
xml.push_str("<VersionId>null</VersionId>");
|
||||||
xml.push_str("<IsLatest>true</IsLatest>");
|
xml.push_str("<IsLatest>true</IsLatest>");
|
||||||
xml.push_str(&format!(
|
xml.push_str(&format!(
|
||||||
@@ -1076,9 +1105,32 @@ pub async fn list_object_versions(state: &AppState, bucket: &str) -> Response {
|
|||||||
myfsio_xml::response::format_s3_datetime(&obj.last_modified)
|
myfsio_xml::response::format_s3_datetime(&obj.last_modified)
|
||||||
));
|
));
|
||||||
if let Some(ref etag) = obj.etag {
|
if let Some(ref etag) = obj.etag {
|
||||||
xml.push_str(&format!("<ETag>\"{}\"</ETag>", etag));
|
xml.push_str(&format!("<ETag>\"{}\"</ETag>", xml_escape(etag)));
|
||||||
}
|
}
|
||||||
xml.push_str(&format!("<Size>{}</Size>", obj.size));
|
xml.push_str(&format!("<Size>{}</Size>", obj.size));
|
||||||
|
xml.push_str(&format!(
|
||||||
|
"<StorageClass>{}</StorageClass>",
|
||||||
|
xml_escape(obj.storage_class.as_deref().unwrap_or("STANDARD"))
|
||||||
|
));
|
||||||
|
xml.push_str("</Version>");
|
||||||
|
}
|
||||||
|
|
||||||
|
for version in archived_versions.iter().take(archived_count) {
|
||||||
|
xml.push_str("<Version>");
|
||||||
|
xml.push_str(&format!("<Key>{}</Key>", xml_escape(&version.key)));
|
||||||
|
xml.push_str(&format!(
|
||||||
|
"<VersionId>{}</VersionId>",
|
||||||
|
xml_escape(&version.version_id)
|
||||||
|
));
|
||||||
|
xml.push_str("<IsLatest>false</IsLatest>");
|
||||||
|
xml.push_str(&format!(
|
||||||
|
"<LastModified>{}</LastModified>",
|
||||||
|
myfsio_xml::response::format_s3_datetime(&version.last_modified)
|
||||||
|
));
|
||||||
|
if let Some(ref etag) = version.etag {
|
||||||
|
xml.push_str(&format!("<ETag>\"{}\"</ETag>", xml_escape(etag)));
|
||||||
|
}
|
||||||
|
xml.push_str(&format!("<Size>{}</Size>", version.size));
|
||||||
xml.push_str("<StorageClass>STANDARD</StorageClass>");
|
xml.push_str("<StorageClass>STANDARD</StorageClass>");
|
||||||
xml.push_str("</Version>");
|
xml.push_str("</Version>");
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -117,6 +117,7 @@ fn storage_status(err: &StorageError) -> StatusCode {
|
|||||||
match err {
|
match err {
|
||||||
StorageError::BucketNotFound(_)
|
StorageError::BucketNotFound(_)
|
||||||
| StorageError::ObjectNotFound { .. }
|
| StorageError::ObjectNotFound { .. }
|
||||||
|
| StorageError::VersionNotFound { .. }
|
||||||
| StorageError::UploadNotFound(_) => StatusCode::NOT_FOUND,
|
| StorageError::UploadNotFound(_) => StatusCode::NOT_FOUND,
|
||||||
StorageError::InvalidBucketName(_)
|
StorageError::InvalidBucketName(_)
|
||||||
| StorageError::InvalidObjectKey(_)
|
| StorageError::InvalidObjectKey(_)
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ pub mod templates;
|
|||||||
|
|
||||||
use axum::Router;
|
use axum::Router;
|
||||||
|
|
||||||
pub const SERVER_HEADER: &str = "MyFSIO";
|
pub const SERVER_HEADER: &str = concat!("MyFSIO-Rust/", env!("CARGO_PKG_VERSION"));
|
||||||
|
|
||||||
pub fn create_ui_router(state: state::AppState) -> Router {
|
pub fn create_ui_router(state: state::AppState) -> Router {
|
||||||
use axum::routing::{delete, get, post, put};
|
use axum::routing::{delete, get, post, put};
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
use axum::extract::{Request, State};
|
use axum::extract::{Request, State};
|
||||||
use axum::http::{header, HeaderMap, Method, StatusCode};
|
use axum::http::{header, HeaderMap, Method, StatusCode, Uri};
|
||||||
use axum::middleware::Next;
|
use axum::middleware::Next;
|
||||||
use axum::response::{IntoResponse, Response};
|
use axum::response::{IntoResponse, Response};
|
||||||
|
|
||||||
@@ -15,6 +15,9 @@ use tokio::io::AsyncReadExt;
|
|||||||
use crate::services::acl::acl_from_bucket_config;
|
use crate::services::acl::acl_from_bucket_config;
|
||||||
use crate::state::AppState;
|
use crate::state::AppState;
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
struct OriginalCanonicalPath(String);
|
||||||
|
|
||||||
fn website_error_response(
|
fn website_error_response(
|
||||||
status: StatusCode,
|
status: StatusCode,
|
||||||
body: Option<Vec<u8>>,
|
body: Option<Vec<u8>>,
|
||||||
@@ -45,7 +48,7 @@ fn website_error_response(
|
|||||||
fn default_website_error_body(status: StatusCode) -> String {
|
fn default_website_error_body(status: StatusCode) -> String {
|
||||||
let code = status.as_u16();
|
let code = status.as_u16();
|
||||||
if status == StatusCode::NOT_FOUND {
|
if status == StatusCode::NOT_FOUND {
|
||||||
"404 page not found".to_string()
|
"<h1>404 page not found</h1>".to_string()
|
||||||
} else {
|
} else {
|
||||||
let reason = status.canonical_reason().unwrap_or("Error");
|
let reason = status.canonical_reason().unwrap_or("Error");
|
||||||
format!("{code} {reason}")
|
format!("{code} {reason}")
|
||||||
@@ -324,6 +327,67 @@ async fn maybe_serve_website(
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn virtual_host_candidate(host: &str) -> Option<String> {
|
||||||
|
let (candidate, _) = host.split_once('.')?;
|
||||||
|
if candidate.is_empty() || matches!(candidate, "www" | "s3" | "api" | "admin" | "kms") {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if myfsio_storage::validation::validate_bucket_name(candidate).is_some() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(candidate.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn virtual_host_bucket(
|
||||||
|
state: &AppState,
|
||||||
|
host: &str,
|
||||||
|
path: &str,
|
||||||
|
method: &Method,
|
||||||
|
) -> Option<String> {
|
||||||
|
if path.starts_with("/ui")
|
||||||
|
|| path.starts_with("/admin")
|
||||||
|
|| path.starts_with("/kms")
|
||||||
|
|| path.starts_with("/myfsio")
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let bucket = virtual_host_candidate(host)?;
|
||||||
|
if path == format!("/{}", bucket) || path.starts_with(&format!("/{}/", bucket)) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
match state.storage.bucket_exists(&bucket).await {
|
||||||
|
Ok(true) => Some(bucket),
|
||||||
|
Ok(false) if *method == Method::PUT && path == "/" => Some(bucket),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rewrite_uri_for_virtual_host(uri: &Uri, bucket: &str) -> Option<Uri> {
|
||||||
|
let path = uri.path();
|
||||||
|
let rewritten_path = if path == "/" {
|
||||||
|
format!("/{}/", bucket)
|
||||||
|
} else {
|
||||||
|
format!("/{}{}", bucket, path)
|
||||||
|
};
|
||||||
|
let path_and_query = match uri.query() {
|
||||||
|
Some(query) => format!("{}?{}", rewritten_path, query),
|
||||||
|
None => rewritten_path,
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut parts = uri.clone().into_parts();
|
||||||
|
parts.path_and_query = Some(path_and_query.parse().ok()?);
|
||||||
|
Uri::from_parts(parts).ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sigv4_canonical_path(req: &Request) -> &str {
|
||||||
|
req.extensions()
|
||||||
|
.get::<OriginalCanonicalPath>()
|
||||||
|
.map(|path| path.0.as_str())
|
||||||
|
.unwrap_or_else(|| req.uri().path())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn auth_layer(State(state): State<AppState>, mut req: Request, next: Next) -> Response {
|
pub async fn auth_layer(State(state): State<AppState>, mut req: Request, next: Next) -> Response {
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
let uri = req.uri().clone();
|
let uri = req.uri().clone();
|
||||||
@@ -360,7 +424,7 @@ pub async fn auth_layer(State(state): State<AppState>, mut req: Request, next: N
|
|||||||
} else if let Some(response) = maybe_serve_website(
|
} else if let Some(response) = maybe_serve_website(
|
||||||
&state,
|
&state,
|
||||||
method.clone(),
|
method.clone(),
|
||||||
host.unwrap_or_default(),
|
host.clone().unwrap_or_default(),
|
||||||
path.clone(),
|
path.clone(),
|
||||||
range_header,
|
range_header,
|
||||||
)
|
)
|
||||||
@@ -368,38 +432,53 @@ pub async fn auth_layer(State(state): State<AppState>, mut req: Request, next: N
|
|||||||
{
|
{
|
||||||
response
|
response
|
||||||
} else {
|
} else {
|
||||||
|
let auth_path = if let Some(bucket) =
|
||||||
|
virtual_host_bucket(&state, host.as_deref().unwrap_or_default(), &path, &method).await
|
||||||
|
{
|
||||||
|
if let Some(rewritten) = rewrite_uri_for_virtual_host(req.uri(), &bucket) {
|
||||||
|
req.extensions_mut()
|
||||||
|
.insert(OriginalCanonicalPath(path.clone()));
|
||||||
|
*req.uri_mut() = rewritten;
|
||||||
|
req.uri().path().to_string()
|
||||||
|
} else {
|
||||||
|
path.clone()
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
path.clone()
|
||||||
|
};
|
||||||
|
|
||||||
match try_auth(&state, &req) {
|
match try_auth(&state, &req) {
|
||||||
AuthResult::NoAuth => match authorize_request(
|
AuthResult::NoAuth => match authorize_request(
|
||||||
&state,
|
&state,
|
||||||
None,
|
None,
|
||||||
&method,
|
&method,
|
||||||
&path,
|
&auth_path,
|
||||||
&query,
|
&query,
|
||||||
copy_source.as_deref(),
|
copy_source.as_deref(),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(()) => next.run(req).await,
|
Ok(()) => next.run(req).await,
|
||||||
Err(err) => error_response(err, &path),
|
Err(err) => error_response(err, &auth_path),
|
||||||
},
|
},
|
||||||
AuthResult::Ok(principal) => {
|
AuthResult::Ok(principal) => {
|
||||||
if let Err(err) = authorize_request(
|
if let Err(err) = authorize_request(
|
||||||
&state,
|
&state,
|
||||||
Some(&principal),
|
Some(&principal),
|
||||||
&method,
|
&method,
|
||||||
&path,
|
&auth_path,
|
||||||
&query,
|
&query,
|
||||||
copy_source.as_deref(),
|
copy_source.as_deref(),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
error_response(err, &path)
|
error_response(err, &auth_path)
|
||||||
} else {
|
} else {
|
||||||
req.extensions_mut().insert(principal);
|
req.extensions_mut().insert(principal);
|
||||||
next.run(req).await
|
next.run(req).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
AuthResult::Denied(err) => error_response(err, &path),
|
AuthResult::Denied(err) => error_response(err, &auth_path),
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1078,7 +1157,7 @@ fn verify_sigv4_header(state: &AppState, req: &Request, auth_str: &str) -> AuthR
|
|||||||
};
|
};
|
||||||
|
|
||||||
let method = req.method().as_str();
|
let method = req.method().as_str();
|
||||||
let canonical_uri = req.uri().path();
|
let canonical_uri = sigv4_canonical_path(req);
|
||||||
|
|
||||||
let query_params = parse_query_params(req.uri().query().unwrap_or(""));
|
let query_params = parse_query_params(req.uri().query().unwrap_or(""));
|
||||||
|
|
||||||
@@ -1234,7 +1313,7 @@ fn verify_sigv4_query(state: &AppState, req: &Request) -> AuthResult {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let method = req.method().as_str();
|
let method = req.method().as_str();
|
||||||
let canonical_uri = req.uri().path();
|
let canonical_uri = sigv4_canonical_path(req);
|
||||||
|
|
||||||
let query_params_no_sig: Vec<(String, String)> = params
|
let query_params_no_sig: Vec<(String, String)> = params
|
||||||
.iter()
|
.iter()
|
||||||
|
|||||||
@@ -2121,6 +2121,445 @@ async fn test_bucket_versioning() {
|
|||||||
assert!(body.contains("<Status>Enabled</Status>"));
|
assert!(body.contains("<Status>Enabled</Status>"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_versioned_object_can_be_read_and_deleted_by_version_id() {
|
||||||
|
let (app, _tmp) = test_app();
|
||||||
|
|
||||||
|
app.clone()
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::PUT,
|
||||||
|
"/versions-bucket",
|
||||||
|
Body::empty(),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
app.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method(Method::PUT)
|
||||||
|
.uri("/versions-bucket?versioning")
|
||||||
|
.header("x-access-key", TEST_ACCESS_KEY)
|
||||||
|
.header("x-secret-key", TEST_SECRET_KEY)
|
||||||
|
.body(Body::from(
|
||||||
|
"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>",
|
||||||
|
))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
app.clone()
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::PUT,
|
||||||
|
"/versions-bucket/doc.txt",
|
||||||
|
Body::from("first"),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
app.clone()
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::PUT,
|
||||||
|
"/versions-bucket/doc.txt",
|
||||||
|
Body::from("second"),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let list_resp = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::GET,
|
||||||
|
"/versions-bucket?versions",
|
||||||
|
Body::empty(),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(list_resp.status(), StatusCode::OK);
|
||||||
|
let list_body = String::from_utf8(
|
||||||
|
list_resp
|
||||||
|
.into_body()
|
||||||
|
.collect()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.to_bytes()
|
||||||
|
.to_vec(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let archived_version_id = list_body
|
||||||
|
.split("<VersionId>")
|
||||||
|
.filter_map(|part| part.split_once("</VersionId>").map(|(id, _)| id))
|
||||||
|
.find(|id| *id != "null")
|
||||||
|
.expect("archived version id")
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
let version_resp = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::GET,
|
||||||
|
&format!("/versions-bucket/doc.txt?versionId={}", archived_version_id),
|
||||||
|
Body::empty(),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(version_resp.status(), StatusCode::OK);
|
||||||
|
assert_eq!(
|
||||||
|
version_resp.headers()["x-amz-version-id"].to_str().unwrap(),
|
||||||
|
archived_version_id
|
||||||
|
);
|
||||||
|
let version_body = version_resp.into_body().collect().await.unwrap().to_bytes();
|
||||||
|
assert_eq!(&version_body[..], b"first");
|
||||||
|
|
||||||
|
let traversal_resp = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::GET,
|
||||||
|
&format!(
|
||||||
|
"/versions-bucket/doc.txt?versionId=../other/{}",
|
||||||
|
archived_version_id
|
||||||
|
),
|
||||||
|
Body::empty(),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(traversal_resp.status(), StatusCode::NOT_FOUND);
|
||||||
|
|
||||||
|
app.clone()
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::PUT,
|
||||||
|
"/versions-bucket/doc.txt",
|
||||||
|
Body::from("third"),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let limited_resp = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::GET,
|
||||||
|
"/versions-bucket?versions&max-keys=1",
|
||||||
|
Body::empty(),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(limited_resp.status(), StatusCode::OK);
|
||||||
|
let limited_body = String::from_utf8(
|
||||||
|
limited_resp
|
||||||
|
.into_body()
|
||||||
|
.collect()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.to_bytes()
|
||||||
|
.to_vec(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(limited_body.matches("<Version>").count(), 1);
|
||||||
|
assert!(limited_body.contains("<IsTruncated>true</IsTruncated>"));
|
||||||
|
|
||||||
|
let delete_resp = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::DELETE,
|
||||||
|
&format!("/versions-bucket/doc.txt?versionId={}", archived_version_id),
|
||||||
|
Body::empty(),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(delete_resp.status(), StatusCode::NO_CONTENT);
|
||||||
|
|
||||||
|
let missing_resp = app
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::GET,
|
||||||
|
&format!("/versions-bucket/doc.txt?versionId={}", archived_version_id),
|
||||||
|
Body::empty(),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(missing_resp.status(), StatusCode::NOT_FOUND);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_retention_is_enforced_when_deleting_archived_version() {
|
||||||
|
let (app, _tmp) = test_app();
|
||||||
|
|
||||||
|
app.clone()
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::PUT,
|
||||||
|
"/locked-versions",
|
||||||
|
Body::empty(),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
app.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method(Method::PUT)
|
||||||
|
.uri("/locked-versions?versioning")
|
||||||
|
.header("x-access-key", TEST_ACCESS_KEY)
|
||||||
|
.header("x-secret-key", TEST_SECRET_KEY)
|
||||||
|
.body(Body::from(
|
||||||
|
"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>",
|
||||||
|
))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
app.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method(Method::PUT)
|
||||||
|
.uri("/locked-versions/doc.txt")
|
||||||
|
.header("x-access-key", TEST_ACCESS_KEY)
|
||||||
|
.header("x-secret-key", TEST_SECRET_KEY)
|
||||||
|
.header("x-amz-object-lock-mode", "GOVERNANCE")
|
||||||
|
.header(
|
||||||
|
"x-amz-object-lock-retain-until-date",
|
||||||
|
"2099-01-01T00:00:00Z",
|
||||||
|
)
|
||||||
|
.body(Body::from("locked"))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
app.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method(Method::PUT)
|
||||||
|
.uri("/locked-versions/doc.txt")
|
||||||
|
.header("x-access-key", TEST_ACCESS_KEY)
|
||||||
|
.header("x-secret-key", TEST_SECRET_KEY)
|
||||||
|
.header("x-amz-bypass-governance-retention", "true")
|
||||||
|
.body(Body::from("replacement"))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let list_resp = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::GET,
|
||||||
|
"/locked-versions?versions",
|
||||||
|
Body::empty(),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(list_resp.status(), StatusCode::OK);
|
||||||
|
let list_body = String::from_utf8(
|
||||||
|
list_resp
|
||||||
|
.into_body()
|
||||||
|
.collect()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.to_bytes()
|
||||||
|
.to_vec(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let archived_version_id = list_body
|
||||||
|
.split("<VersionId>")
|
||||||
|
.filter_map(|part| part.split_once("</VersionId>").map(|(id, _)| id))
|
||||||
|
.find(|id| *id != "null")
|
||||||
|
.expect("archived version id")
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
let denied = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::DELETE,
|
||||||
|
&format!("/locked-versions/doc.txt?versionId={}", archived_version_id),
|
||||||
|
Body::empty(),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(denied.status(), StatusCode::FORBIDDEN);
|
||||||
|
|
||||||
|
let allowed = app
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method(Method::DELETE)
|
||||||
|
.uri(format!(
|
||||||
|
"/locked-versions/doc.txt?versionId={}",
|
||||||
|
archived_version_id
|
||||||
|
))
|
||||||
|
.header("x-access-key", TEST_ACCESS_KEY)
|
||||||
|
.header("x-secret-key", TEST_SECRET_KEY)
|
||||||
|
.header("x-amz-bypass-governance-retention", "true")
|
||||||
|
.body(Body::empty())
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(allowed.status(), StatusCode::NO_CONTENT);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_put_object_validates_content_md5() {
|
||||||
|
let (app, _tmp) = test_app();
|
||||||
|
|
||||||
|
app.clone()
|
||||||
|
.oneshot(signed_request(Method::PUT, "/md5-bucket", Body::empty()))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let bad_resp = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method(Method::PUT)
|
||||||
|
.uri("/md5-bucket/object.txt")
|
||||||
|
.header("x-access-key", TEST_ACCESS_KEY)
|
||||||
|
.header("x-secret-key", TEST_SECRET_KEY)
|
||||||
|
.header("content-md5", "AAAAAAAAAAAAAAAAAAAAAA==")
|
||||||
|
.body(Body::from("hello"))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(bad_resp.status(), StatusCode::BAD_REQUEST);
|
||||||
|
let bad_body = String::from_utf8(
|
||||||
|
bad_resp
|
||||||
|
.into_body()
|
||||||
|
.collect()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.to_bytes()
|
||||||
|
.to_vec(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(bad_body.contains("<Code>BadDigest</Code>"));
|
||||||
|
|
||||||
|
let good_resp = app
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method(Method::PUT)
|
||||||
|
.uri("/md5-bucket/object.txt")
|
||||||
|
.header("x-access-key", TEST_ACCESS_KEY)
|
||||||
|
.header("x-secret-key", TEST_SECRET_KEY)
|
||||||
|
.header("content-md5", "XUFAKrxLKna5cZ2REBfFkg==")
|
||||||
|
.body(Body::from("hello"))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(good_resp.status(), StatusCode::OK);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_put_object_tagging_and_standard_headers_are_persisted() {
|
||||||
|
let (app, _tmp) = test_app();
|
||||||
|
|
||||||
|
app.clone()
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::PUT,
|
||||||
|
"/headers-bucket",
|
||||||
|
Body::empty(),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let put_resp = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method(Method::PUT)
|
||||||
|
.uri("/headers-bucket/report.txt")
|
||||||
|
.header("x-access-key", TEST_ACCESS_KEY)
|
||||||
|
.header("x-secret-key", TEST_SECRET_KEY)
|
||||||
|
.header("x-amz-tagging", "env=prod&name=quarter%201")
|
||||||
|
.header("cache-control", "max-age=60")
|
||||||
|
.header("content-disposition", "attachment")
|
||||||
|
.header("content-language", "en-US")
|
||||||
|
.header("x-amz-storage-class", "STANDARD_IA")
|
||||||
|
.body(Body::from("report"))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(put_resp.status(), StatusCode::OK);
|
||||||
|
|
||||||
|
let head_resp = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::HEAD,
|
||||||
|
"/headers-bucket/report.txt",
|
||||||
|
Body::empty(),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(head_resp.status(), StatusCode::OK);
|
||||||
|
assert_eq!(head_resp.headers()["cache-control"], "max-age=60");
|
||||||
|
assert_eq!(head_resp.headers()["content-disposition"], "attachment");
|
||||||
|
assert_eq!(head_resp.headers()["content-language"], "en-US");
|
||||||
|
assert_eq!(head_resp.headers()["x-amz-storage-class"], "STANDARD_IA");
|
||||||
|
|
||||||
|
let tags_resp = app
|
||||||
|
.oneshot(signed_request(
|
||||||
|
Method::GET,
|
||||||
|
"/headers-bucket/report.txt?tagging",
|
||||||
|
Body::empty(),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(tags_resp.status(), StatusCode::OK);
|
||||||
|
let tags_body = String::from_utf8(
|
||||||
|
tags_resp
|
||||||
|
.into_body()
|
||||||
|
.collect()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.to_bytes()
|
||||||
|
.to_vec(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(tags_body.contains("<Key>env</Key>"));
|
||||||
|
assert!(tags_body.contains("<Value>prod</Value>"));
|
||||||
|
assert!(tags_body.contains("<Key>name</Key>"));
|
||||||
|
assert!(tags_body.contains("<Value>quarter 1</Value>"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_virtual_host_bucket_routes_to_s3_object_handlers() {
|
||||||
|
let (app, _tmp) = test_app();
|
||||||
|
|
||||||
|
app.clone()
|
||||||
|
.oneshot(signed_request(Method::PUT, "/vh-bucket", Body::empty()))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let put_resp = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method(Method::PUT)
|
||||||
|
.uri("/hello.txt")
|
||||||
|
.header("host", "vh-bucket.localhost")
|
||||||
|
.header("x-access-key", TEST_ACCESS_KEY)
|
||||||
|
.header("x-secret-key", TEST_SECRET_KEY)
|
||||||
|
.body(Body::from("virtual host body"))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(put_resp.status(), StatusCode::OK);
|
||||||
|
|
||||||
|
let get_resp = app
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method(Method::GET)
|
||||||
|
.uri("/hello.txt")
|
||||||
|
.header("host", "vh-bucket.localhost")
|
||||||
|
.header("x-access-key", TEST_ACCESS_KEY)
|
||||||
|
.header("x-secret-key", TEST_SECRET_KEY)
|
||||||
|
.body(Body::empty())
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(get_resp.status(), StatusCode::OK);
|
||||||
|
let body = get_resp.into_body().collect().await.unwrap().to_bytes();
|
||||||
|
assert_eq!(&body[..], b"virtual host body");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_bucket_tagging() {
|
async fn test_bucket_tagging() {
|
||||||
let (app, _tmp) = test_app();
|
let (app, _tmp) = test_app();
|
||||||
@@ -3323,7 +3762,7 @@ async fn test_static_website_default_404_returns_html_body() {
|
|||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(body.len(), content_length);
|
assert_eq!(body.len(), content_length);
|
||||||
assert_eq!(body, "404 page not found");
|
assert_eq!(body, "<h1>404 page not found</h1>");
|
||||||
|
|
||||||
let head_resp = app
|
let head_resp = app
|
||||||
.oneshot(website_request(Method::HEAD, "/missing.html"))
|
.oneshot(website_request(Method::HEAD, "/missing.html"))
|
||||||
|
|||||||
@@ -11,6 +11,12 @@ pub enum StorageError {
|
|||||||
BucketNotEmpty(String),
|
BucketNotEmpty(String),
|
||||||
#[error("Object not found: {bucket}/{key}")]
|
#[error("Object not found: {bucket}/{key}")]
|
||||||
ObjectNotFound { bucket: String, key: String },
|
ObjectNotFound { bucket: String, key: String },
|
||||||
|
#[error("Object version not found: {bucket}/{key}?versionId={version_id}")]
|
||||||
|
VersionNotFound {
|
||||||
|
bucket: String,
|
||||||
|
key: String,
|
||||||
|
version_id: String,
|
||||||
|
},
|
||||||
#[error("Invalid bucket name: {0}")]
|
#[error("Invalid bucket name: {0}")]
|
||||||
InvalidBucketName(String),
|
InvalidBucketName(String),
|
||||||
#[error("Invalid object key: {0}")]
|
#[error("Invalid object key: {0}")]
|
||||||
@@ -46,6 +52,12 @@ impl From<StorageError> for S3Error {
|
|||||||
S3Error::from_code(S3ErrorCode::NoSuchKey)
|
S3Error::from_code(S3ErrorCode::NoSuchKey)
|
||||||
.with_resource(format!("/{}/{}", bucket, key))
|
.with_resource(format!("/{}/{}", bucket, key))
|
||||||
}
|
}
|
||||||
|
StorageError::VersionNotFound {
|
||||||
|
bucket,
|
||||||
|
key,
|
||||||
|
version_id,
|
||||||
|
} => S3Error::from_code(S3ErrorCode::NoSuchVersion)
|
||||||
|
.with_resource(format!("/{}/{}?versionId={}", bucket, key, version_id)),
|
||||||
StorageError::InvalidBucketName(msg) => {
|
StorageError::InvalidBucketName(msg) => {
|
||||||
S3Error::new(S3ErrorCode::InvalidBucketName, msg)
|
S3Error::new(S3ErrorCode::InvalidBucketName, msg)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -605,6 +605,144 @@ impl FsStorageBackend {
|
|||||||
Ok(source_size)
|
Ok(source_size)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn version_record_paths(
|
||||||
|
&self,
|
||||||
|
bucket_name: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
) -> (PathBuf, PathBuf) {
|
||||||
|
let version_dir = self.version_dir(bucket_name, key);
|
||||||
|
(
|
||||||
|
version_dir.join(format!("{}.json", version_id)),
|
||||||
|
version_dir.join(format!("{}.bin", version_id)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_version_id(bucket_name: &str, key: &str, version_id: &str) -> StorageResult<()> {
|
||||||
|
if version_id.is_empty()
|
||||||
|
|| version_id.contains('/')
|
||||||
|
|| version_id.contains('\\')
|
||||||
|
|| version_id.contains("..")
|
||||||
|
{
|
||||||
|
return Err(StorageError::VersionNotFound {
|
||||||
|
bucket: bucket_name.to_string(),
|
||||||
|
key: key.to_string(),
|
||||||
|
version_id: version_id.to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_version_record_sync(
|
||||||
|
&self,
|
||||||
|
bucket_name: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
) -> StorageResult<(Value, PathBuf)> {
|
||||||
|
self.require_bucket(bucket_name)?;
|
||||||
|
self.validate_key(key)?;
|
||||||
|
Self::validate_version_id(bucket_name, key, version_id)?;
|
||||||
|
let (manifest_path, data_path) = self.version_record_paths(bucket_name, key, version_id);
|
||||||
|
if !manifest_path.is_file() || !data_path.is_file() {
|
||||||
|
return Err(StorageError::VersionNotFound {
|
||||||
|
bucket: bucket_name.to_string(),
|
||||||
|
key: key.to_string(),
|
||||||
|
version_id: version_id.to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let content = std::fs::read_to_string(&manifest_path).map_err(StorageError::Io)?;
|
||||||
|
let record = serde_json::from_str::<Value>(&content).map_err(StorageError::Json)?;
|
||||||
|
Ok((record, data_path))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn version_metadata_from_record(record: &Value) -> HashMap<String, String> {
|
||||||
|
record
|
||||||
|
.get("metadata")
|
||||||
|
.and_then(Value::as_object)
|
||||||
|
.map(|meta| {
|
||||||
|
meta.iter()
|
||||||
|
.filter_map(|(k, v)| v.as_str().map(|s| (k.clone(), s.to_string())))
|
||||||
|
.collect::<HashMap<String, String>>()
|
||||||
|
})
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn object_meta_from_version_record(
|
||||||
|
&self,
|
||||||
|
key: &str,
|
||||||
|
record: &Value,
|
||||||
|
data_path: &Path,
|
||||||
|
) -> StorageResult<ObjectMeta> {
|
||||||
|
let metadata = Self::version_metadata_from_record(record);
|
||||||
|
|
||||||
|
let data_len = std::fs::metadata(data_path)
|
||||||
|
.map(|meta| meta.len())
|
||||||
|
.unwrap_or_default();
|
||||||
|
let size = record
|
||||||
|
.get("size")
|
||||||
|
.and_then(Value::as_u64)
|
||||||
|
.unwrap_or(data_len);
|
||||||
|
let last_modified = record
|
||||||
|
.get("archived_at")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.and_then(|value| DateTime::parse_from_rfc3339(value).ok())
|
||||||
|
.map(|value| value.with_timezone(&Utc))
|
||||||
|
.unwrap_or_else(Utc::now);
|
||||||
|
let etag = record
|
||||||
|
.get("etag")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
.or_else(|| metadata.get("__etag__").cloned());
|
||||||
|
|
||||||
|
let mut obj = ObjectMeta::new(key.to_string(), size, last_modified);
|
||||||
|
obj.etag = etag;
|
||||||
|
obj.content_type = metadata.get("__content_type__").cloned();
|
||||||
|
obj.storage_class = metadata
|
||||||
|
.get("__storage_class__")
|
||||||
|
.cloned()
|
||||||
|
.or_else(|| Some("STANDARD".to_string()));
|
||||||
|
obj.metadata = metadata
|
||||||
|
.into_iter()
|
||||||
|
.filter(|(k, _)| !k.starts_with("__"))
|
||||||
|
.collect();
|
||||||
|
Ok(obj)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn version_info_from_record(&self, fallback_key: &str, record: &Value) -> VersionInfo {
|
||||||
|
let version_id = record
|
||||||
|
.get("version_id")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
let key = record
|
||||||
|
.get("key")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or(fallback_key)
|
||||||
|
.to_string();
|
||||||
|
let size = record.get("size").and_then(Value::as_u64).unwrap_or(0);
|
||||||
|
let archived_at = record
|
||||||
|
.get("archived_at")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.and_then(|s| DateTime::parse_from_rfc3339(s).ok())
|
||||||
|
.map(|d| d.with_timezone(&Utc))
|
||||||
|
.unwrap_or_else(Utc::now);
|
||||||
|
let etag = record
|
||||||
|
.get("etag")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
|
||||||
|
VersionInfo {
|
||||||
|
version_id,
|
||||||
|
key,
|
||||||
|
size,
|
||||||
|
last_modified: archived_at,
|
||||||
|
etag,
|
||||||
|
is_latest: false,
|
||||||
|
is_delete_marker: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn bucket_stats_sync(&self, bucket_name: &str) -> StorageResult<BucketStats> {
|
fn bucket_stats_sync(&self, bucket_name: &str) -> StorageResult<BucketStats> {
|
||||||
let bucket_path = self.require_bucket(bucket_name)?;
|
let bucket_path = self.require_bucket(bucket_name)?;
|
||||||
|
|
||||||
@@ -1241,6 +1379,10 @@ impl crate::traits::StorageEngine for FsStorageBackend {
|
|||||||
let mut obj = ObjectMeta::new(key.to_string(), meta.len(), lm);
|
let mut obj = ObjectMeta::new(key.to_string(), meta.len(), lm);
|
||||||
obj.etag = stored_meta.get("__etag__").cloned();
|
obj.etag = stored_meta.get("__etag__").cloned();
|
||||||
obj.content_type = stored_meta.get("__content_type__").cloned();
|
obj.content_type = stored_meta.get("__content_type__").cloned();
|
||||||
|
obj.storage_class = stored_meta
|
||||||
|
.get("__storage_class__")
|
||||||
|
.cloned()
|
||||||
|
.or_else(|| Some("STANDARD".to_string()));
|
||||||
obj.metadata = stored_meta
|
obj.metadata = stored_meta
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|(k, _)| !k.starts_with("__"))
|
.filter(|(k, _)| !k.starts_with("__"))
|
||||||
@@ -1289,6 +1431,10 @@ impl crate::traits::StorageEngine for FsStorageBackend {
|
|||||||
let mut obj = ObjectMeta::new(key.to_string(), meta.len(), lm);
|
let mut obj = ObjectMeta::new(key.to_string(), meta.len(), lm);
|
||||||
obj.etag = stored_meta.get("__etag__").cloned();
|
obj.etag = stored_meta.get("__etag__").cloned();
|
||||||
obj.content_type = stored_meta.get("__content_type__").cloned();
|
obj.content_type = stored_meta.get("__content_type__").cloned();
|
||||||
|
obj.storage_class = stored_meta
|
||||||
|
.get("__storage_class__")
|
||||||
|
.cloned()
|
||||||
|
.or_else(|| Some("STANDARD".to_string()));
|
||||||
obj.metadata = stored_meta
|
obj.metadata = stored_meta
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|(k, _)| !k.starts_with("__"))
|
.filter(|(k, _)| !k.starts_with("__"))
|
||||||
@@ -1296,6 +1442,51 @@ impl crate::traits::StorageEngine for FsStorageBackend {
|
|||||||
Ok(obj)
|
Ok(obj)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn get_object_version(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
) -> StorageResult<(ObjectMeta, AsyncReadStream)> {
|
||||||
|
let (record, data_path) = self.read_version_record_sync(bucket, key, version_id)?;
|
||||||
|
let obj = self.object_meta_from_version_record(key, &record, &data_path)?;
|
||||||
|
let file = tokio::fs::File::open(&data_path)
|
||||||
|
.await
|
||||||
|
.map_err(StorageError::Io)?;
|
||||||
|
let stream: AsyncReadStream = Box::pin(file);
|
||||||
|
Ok((obj, stream))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_object_version_path(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
) -> StorageResult<PathBuf> {
|
||||||
|
let (_record, data_path) = self.read_version_record_sync(bucket, key, version_id)?;
|
||||||
|
Ok(data_path)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn head_object_version(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
) -> StorageResult<ObjectMeta> {
|
||||||
|
let (record, data_path) = self.read_version_record_sync(bucket, key, version_id)?;
|
||||||
|
self.object_meta_from_version_record(key, &record, &data_path)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_object_version_metadata(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
) -> StorageResult<HashMap<String, String>> {
|
||||||
|
let (record, _data_path) = self.read_version_record_sync(bucket, key, version_id)?;
|
||||||
|
Ok(Self::version_metadata_from_record(&record))
|
||||||
|
}
|
||||||
|
|
||||||
async fn delete_object(&self, bucket: &str, key: &str) -> StorageResult<()> {
|
async fn delete_object(&self, bucket: &str, key: &str) -> StorageResult<()> {
|
||||||
let bucket_path = self.require_bucket(bucket)?;
|
let bucket_path = self.require_bucket(bucket)?;
|
||||||
let path = self.object_path(bucket, key)?;
|
let path = self.object_path(bucket, key)?;
|
||||||
@@ -1317,6 +1508,32 @@ impl crate::traits::StorageEngine for FsStorageBackend {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn delete_object_version(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
) -> StorageResult<()> {
|
||||||
|
self.require_bucket(bucket)?;
|
||||||
|
self.validate_key(key)?;
|
||||||
|
Self::validate_version_id(bucket, key, version_id)?;
|
||||||
|
let (manifest_path, data_path) = self.version_record_paths(bucket, key, version_id);
|
||||||
|
if !manifest_path.is_file() && !data_path.is_file() {
|
||||||
|
return Err(StorageError::VersionNotFound {
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
key: key.to_string(),
|
||||||
|
version_id: version_id.to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
Self::safe_unlink(&data_path).map_err(StorageError::Io)?;
|
||||||
|
Self::safe_unlink(&manifest_path).map_err(StorageError::Io)?;
|
||||||
|
let versions_root = self.bucket_versions_root(bucket);
|
||||||
|
Self::cleanup_empty_parents(&manifest_path, &versions_root);
|
||||||
|
self.stats_cache.remove(bucket);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn copy_object(
|
async fn copy_object(
|
||||||
&self,
|
&self,
|
||||||
src_bucket: &str,
|
src_bucket: &str,
|
||||||
@@ -1817,40 +2034,73 @@ impl crate::traits::StorageEngine for FsStorageBackend {
|
|||||||
}
|
}
|
||||||
if let Ok(content) = std::fs::read_to_string(entry.path()) {
|
if let Ok(content) = std::fs::read_to_string(entry.path()) {
|
||||||
if let Ok(record) = serde_json::from_str::<Value>(&content) {
|
if let Ok(record) = serde_json::from_str::<Value>(&content) {
|
||||||
let version_id = record
|
versions.push(self.version_info_from_record(key, &record));
|
||||||
.get("version_id")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.unwrap_or("")
|
|
||||||
.to_string();
|
|
||||||
let size = record.get("size").and_then(|v| v.as_u64()).unwrap_or(0);
|
|
||||||
let archived_at = record
|
|
||||||
.get("archived_at")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.and_then(|s| DateTime::parse_from_rfc3339(s).ok())
|
|
||||||
.map(|d| d.with_timezone(&Utc))
|
|
||||||
.unwrap_or_else(Utc::now);
|
|
||||||
let etag = record
|
|
||||||
.get("etag")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.map(|s| s.to_string());
|
|
||||||
|
|
||||||
versions.push(VersionInfo {
|
|
||||||
version_id,
|
|
||||||
key: key.to_string(),
|
|
||||||
size,
|
|
||||||
last_modified: archived_at,
|
|
||||||
etag,
|
|
||||||
is_latest: false,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
versions.sort_by(|a, b| b.last_modified.cmp(&a.last_modified));
|
versions.sort_by(|a, b| b.last_modified.cmp(&a.last_modified));
|
||||||
if let Some(first) = versions.first_mut() {
|
|
||||||
first.is_latest = true;
|
Ok(versions)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn list_bucket_object_versions(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
prefix: Option<&str>,
|
||||||
|
) -> StorageResult<Vec<VersionInfo>> {
|
||||||
|
self.require_bucket(bucket)?;
|
||||||
|
let root = self.bucket_versions_root(bucket);
|
||||||
|
if !root.exists() {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut versions = Vec::new();
|
||||||
|
let mut stack = vec![root.clone()];
|
||||||
|
while let Some(current) = stack.pop() {
|
||||||
|
let entries = match std::fs::read_dir(¤t) {
|
||||||
|
Ok(entries) => entries,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let path = entry.path();
|
||||||
|
let ft = match entry.file_type() {
|
||||||
|
Ok(ft) => ft,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
if ft.is_dir() {
|
||||||
|
stack.push(path);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !ft.is_file() || path.extension().and_then(|ext| ext.to_str()) != Some("json") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let content = match std::fs::read_to_string(&path) {
|
||||||
|
Ok(content) => content,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
let record = match serde_json::from_str::<Value>(&content) {
|
||||||
|
Ok(record) => record,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
let fallback_key = path
|
||||||
|
.parent()
|
||||||
|
.and_then(|parent| parent.strip_prefix(&root).ok())
|
||||||
|
.map(|rel| rel.to_string_lossy().replace('\\', "/"))
|
||||||
|
.unwrap_or_default();
|
||||||
|
let info = self.version_info_from_record(&fallback_key, &record);
|
||||||
|
if prefix.is_some_and(|value| !info.key.starts_with(value)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
versions.push(info);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
versions.sort_by(|a, b| {
|
||||||
|
a.key
|
||||||
|
.cmp(&b.key)
|
||||||
|
.then_with(|| b.last_modified.cmp(&a.last_modified))
|
||||||
|
});
|
||||||
Ok(versions)
|
Ok(versions)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2271,6 +2521,12 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(versions.len(), 1);
|
assert_eq!(versions.len(), 1);
|
||||||
assert_eq!(versions[0].size, 8);
|
assert_eq!(versions[0].size, 8);
|
||||||
|
|
||||||
|
let invalid_version = format!("../other/{}", versions[0].version_id);
|
||||||
|
let result = backend
|
||||||
|
.get_object_version("test-bucket", "file.txt", &invalid_version)
|
||||||
|
.await;
|
||||||
|
assert!(matches!(result, Err(StorageError::VersionNotFound { .. })));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -34,8 +34,43 @@ pub trait StorageEngine: Send + Sync {
|
|||||||
|
|
||||||
async fn head_object(&self, bucket: &str, key: &str) -> StorageResult<ObjectMeta>;
|
async fn head_object(&self, bucket: &str, key: &str) -> StorageResult<ObjectMeta>;
|
||||||
|
|
||||||
|
async fn get_object_version(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
) -> StorageResult<(ObjectMeta, AsyncReadStream)>;
|
||||||
|
|
||||||
|
async fn get_object_version_path(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
) -> StorageResult<PathBuf>;
|
||||||
|
|
||||||
|
async fn head_object_version(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
) -> StorageResult<ObjectMeta>;
|
||||||
|
|
||||||
|
async fn get_object_version_metadata(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
) -> StorageResult<HashMap<String, String>>;
|
||||||
|
|
||||||
async fn delete_object(&self, bucket: &str, key: &str) -> StorageResult<()>;
|
async fn delete_object(&self, bucket: &str, key: &str) -> StorageResult<()>;
|
||||||
|
|
||||||
|
async fn delete_object_version(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: &str,
|
||||||
|
) -> StorageResult<()>;
|
||||||
|
|
||||||
async fn copy_object(
|
async fn copy_object(
|
||||||
&self,
|
&self,
|
||||||
src_bucket: &str,
|
src_bucket: &str,
|
||||||
@@ -120,6 +155,12 @@ pub trait StorageEngine: Send + Sync {
|
|||||||
key: &str,
|
key: &str,
|
||||||
) -> StorageResult<Vec<VersionInfo>>;
|
) -> StorageResult<Vec<VersionInfo>>;
|
||||||
|
|
||||||
|
async fn list_bucket_object_versions(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
prefix: Option<&str>,
|
||||||
|
) -> StorageResult<Vec<VersionInfo>>;
|
||||||
|
|
||||||
async fn get_object_tags(&self, bucket: &str, key: &str) -> StorageResult<Vec<Tag>>;
|
async fn get_object_tags(&self, bucket: &str, key: &str) -> StorageResult<Vec<Tag>>;
|
||||||
|
|
||||||
async fn set_object_tags(&self, bucket: &str, key: &str, tags: &[Tag]) -> StorageResult<()>;
|
async fn set_object_tags(&self, bucket: &str, key: &str, tags: &[Tag]) -> StorageResult<()>;
|
||||||
|
|||||||
Reference in New Issue
Block a user