kqjy/MyFSIO
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

Add Rust storage engine foundation

Browse Source
This commit is contained in:
kqjy
2026-04-02 17:00:58 +08:00
parent 1eadc7b75c
commit 926a7e6366
48 changed files with 12189 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
myfsio-engine/crates/myfsio-xml/Cargo.toml Normal file
View File

@@ -0,0 +1,10 @@
[package]
name = "myfsio-xml"
version = "0.1.0"
edition = "2021"
[dependencies]
myfsio-common = { path = "../myfsio-common" }
quick-xml = { workspace = true }
serde = { workspace = true }
chrono = { workspace = true }

14
myfsio-engine/crates/myfsio-xml/src/lib.rs Normal file
View File

@@ -0,0 +1,14 @@
pub mod response;
pub mod request;
use quick_xml::Writer;
use std::io::Cursor;
pub fn write_xml_element(tag: &str, text: &str) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer
.create_element(tag)
.write_text_content(quick_xml::events::BytesText::new(text))
.unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}

159
myfsio-engine/crates/myfsio-xml/src/request.rs Normal file
View File

@@ -0,0 +1,159 @@
use quick_xml::events::Event;
use quick_xml::Reader;
#[derive(Debug, Default)]
pub struct DeleteObjectsRequest {
pub objects: Vec<ObjectIdentifier>,
pub quiet: bool,
}
#[derive(Debug)]
pub struct ObjectIdentifier {
pub key: String,
pub version_id: Option<String>,
}
#[derive(Debug, Default)]
pub struct CompleteMultipartUpload {
pub parts: Vec<CompletedPart>,
}
#[derive(Debug)]
pub struct CompletedPart {
pub part_number: u32,
pub etag: String,
}
pub fn parse_complete_multipart_upload(xml: &str) -> Result<CompleteMultipartUpload, String> {
let mut reader = Reader::from_str(xml);
let mut result = CompleteMultipartUpload::default();
let mut buf = Vec::new();
let mut current_tag = String::new();
let mut part_number: Option<u32> = None;
let mut etag: Option<String> = None;
let mut in_part = false;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
let name = String::from_utf8_lossy(e.name().as_ref()).to_string();
current_tag = name.clone();
if name == "Part" {
in_part = true;
part_number = None;
etag = None;
}
}
Ok(Event::Text(ref e)) => {
if in_part {
let text = e.unescape().map_err(|e| e.to_string())?.to_string();
match current_tag.as_str() {
"PartNumber" => {
part_number = Some(text.trim().parse().map_err(|e: std::num::ParseIntError| e.to_string())?);
}
"ETag" => {
etag = Some(text.trim().trim_matches('"').to_string());
}
_ => {}
}
}
}
Ok(Event::End(ref e)) => {
let name = String::from_utf8_lossy(e.name().as_ref()).to_string();
if name == "Part" && in_part {
if let (Some(pn), Some(et)) = (part_number.take(), etag.take()) {
result.parts.push(CompletedPart {
part_number: pn,
etag: et,
});
}
in_part = false;
}
}
Ok(Event::Eof) => break,
Err(e) => return Err(format!("XML parse error: {}", e)),
_ => {}
}
buf.clear();
}
result.parts.sort_by_key(|p| p.part_number);
Ok(result)
}
pub fn parse_delete_objects(xml: &str) -> Result<DeleteObjectsRequest, String> {
let mut reader = Reader::from_str(xml);
let mut result = DeleteObjectsRequest::default();
let mut buf = Vec::new();
let mut current_tag = String::new();
let mut current_key: Option<String> = None;
let mut current_version_id: Option<String> = None;
let mut in_object = false;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
let name = String::from_utf8_lossy(e.name().as_ref()).to_string();
current_tag = name.clone();
if name == "Object" {
in_object = true;
current_key = None;
current_version_id = None;
}
}
Ok(Event::Text(ref e)) => {
let text = e.unescape().map_err(|e| e.to_string())?.to_string();
match current_tag.as_str() {
"Key" if in_object => {
current_key = Some(text.trim().to_string());
}
"VersionId" if in_object => {
current_version_id = Some(text.trim().to_string());
}
"Quiet" => {
result.quiet = text.trim() == "true";
}
_ => {}
}
}
Ok(Event::End(ref e)) => {
let name = String::from_utf8_lossy(e.name().as_ref()).to_string();
if name == "Object" && in_object {
if let Some(key) = current_key.take() {
result.objects.push(ObjectIdentifier {
key,
version_id: current_version_id.take(),
});
}
in_object = false;
}
}
Ok(Event::Eof) => break,
Err(e) => return Err(format!("XML parse error: {}", e)),
_ => {}
}
buf.clear();
}
Ok(result)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_complete_multipart() {
let xml = r#"<CompleteMultipartUpload>
<Part><PartNumber>2</PartNumber><ETag>"etag2"</ETag></Part>
<Part><PartNumber>1</PartNumber><ETag>"etag1"</ETag></Part>
</CompleteMultipartUpload>"#;
let result = parse_complete_multipart_upload(xml).unwrap();
assert_eq!(result.parts.len(), 2);
assert_eq!(result.parts[0].part_number, 1);
assert_eq!(result.parts[0].etag, "etag1");
assert_eq!(result.parts[1].part_number, 2);
assert_eq!(result.parts[1].etag, "etag2");
}
}

269
myfsio-engine/crates/myfsio-xml/src/response.rs Normal file
View File

@@ -0,0 +1,269 @@
use myfsio_common::types::{BucketMeta, ObjectMeta};
use quick_xml::events::{BytesDecl, BytesEnd, BytesStart, BytesText, Event};
use quick_xml::Writer;
use std::io::Cursor;
pub fn list_buckets_xml(owner_id: &str, owner_name: &str, buckets: &[BucketMeta]) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
let start = BytesStart::new("ListAllMyBucketsResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
writer.write_event(Event::Start(start)).unwrap();
writer.write_event(Event::Start(BytesStart::new("Owner"))).unwrap();
write_text_element(&mut writer, "ID", owner_id);
write_text_element(&mut writer, "DisplayName", owner_name);
writer.write_event(Event::End(BytesEnd::new("Owner"))).unwrap();
writer.write_event(Event::Start(BytesStart::new("Buckets"))).unwrap();
for bucket in buckets {
writer.write_event(Event::Start(BytesStart::new("Bucket"))).unwrap();
write_text_element(&mut writer, "Name", &bucket.name);
write_text_element(&mut writer, "CreationDate", &bucket.creation_date.to_rfc3339());
writer.write_event(Event::End(BytesEnd::new("Bucket"))).unwrap();
}
writer.write_event(Event::End(BytesEnd::new("Buckets"))).unwrap();
writer.write_event(Event::End(BytesEnd::new("ListAllMyBucketsResult"))).unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
pub fn list_objects_v2_xml(
bucket_name: &str,
prefix: &str,
delimiter: &str,
max_keys: usize,
objects: &[ObjectMeta],
common_prefixes: &[String],
is_truncated: bool,
continuation_token: Option<&str>,
next_continuation_token: Option<&str>,
key_count: usize,
) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
let start = BytesStart::new("ListBucketResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
writer.write_event(Event::Start(start)).unwrap();
write_text_element(&mut writer, "Name", bucket_name);
write_text_element(&mut writer, "Prefix", prefix);
if !delimiter.is_empty() {
write_text_element(&mut writer, "Delimiter", delimiter);
}
write_text_element(&mut writer, "MaxKeys", &max_keys.to_string());
write_text_element(&mut writer, "KeyCount", &key_count.to_string());
write_text_element(&mut writer, "IsTruncated", &is_truncated.to_string());
if let Some(token) = continuation_token {
write_text_element(&mut writer, "ContinuationToken", token);
}
if let Some(token) = next_continuation_token {
write_text_element(&mut writer, "NextContinuationToken", token);
}
for obj in objects {
writer.write_event(Event::Start(BytesStart::new("Contents"))).unwrap();
write_text_element(&mut writer, "Key", &obj.key);
write_text_element(&mut writer, "LastModified", &obj.last_modified.to_rfc3339());
if let Some(ref etag) = obj.etag {
write_text_element(&mut writer, "ETag", &format!("\"{}\"", etag));
}
write_text_element(&mut writer, "Size", &obj.size.to_string());
write_text_element(&mut writer, "StorageClass", obj.storage_class.as_deref().unwrap_or("STANDARD"));
writer.write_event(Event::End(BytesEnd::new("Contents"))).unwrap();
}
for prefix in common_prefixes {
writer.write_event(Event::Start(BytesStart::new("CommonPrefixes"))).unwrap();
write_text_element(&mut writer, "Prefix", prefix);
writer.write_event(Event::End(BytesEnd::new("CommonPrefixes"))).unwrap();
}
writer.write_event(Event::End(BytesEnd::new("ListBucketResult"))).unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
fn write_text_element(writer: &mut Writer<Cursor<Vec<u8>>>, tag: &str, text: &str) {
writer.write_event(Event::Start(BytesStart::new(tag))).unwrap();
writer.write_event(Event::Text(BytesText::new(text))).unwrap();
writer.write_event(Event::End(BytesEnd::new(tag))).unwrap();
}
pub fn initiate_multipart_upload_xml(bucket: &str, key: &str, upload_id: &str) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
let start = BytesStart::new("InitiateMultipartUploadResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
writer.write_event(Event::Start(start)).unwrap();
write_text_element(&mut writer, "Bucket", bucket);
write_text_element(&mut writer, "Key", key);
write_text_element(&mut writer, "UploadId", upload_id);
writer.write_event(Event::End(BytesEnd::new("InitiateMultipartUploadResult"))).unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
pub fn complete_multipart_upload_xml(
bucket: &str,
key: &str,
etag: &str,
location: &str,
) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
let start = BytesStart::new("CompleteMultipartUploadResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
writer.write_event(Event::Start(start)).unwrap();
write_text_element(&mut writer, "Location", location);
write_text_element(&mut writer, "Bucket", bucket);
write_text_element(&mut writer, "Key", key);
write_text_element(&mut writer, "ETag", &format!("\"{}\"", etag));
writer.write_event(Event::End(BytesEnd::new("CompleteMultipartUploadResult"))).unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
pub fn copy_object_result_xml(etag: &str, last_modified: &str) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
let start = BytesStart::new("CopyObjectResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
writer.write_event(Event::Start(start)).unwrap();
write_text_element(&mut writer, "ETag", &format!("\"{}\"", etag));
write_text_element(&mut writer, "LastModified", last_modified);
writer.write_event(Event::End(BytesEnd::new("CopyObjectResult"))).unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
pub fn delete_result_xml(
deleted: &[(String, Option<String>)],
errors: &[(String, String, String)],
quiet: bool,
) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
let start = BytesStart::new("DeleteResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
writer.write_event(Event::Start(start)).unwrap();
if !quiet {
for (key, version_id) in deleted {
writer.write_event(Event::Start(BytesStart::new("Deleted"))).unwrap();
write_text_element(&mut writer, "Key", key);
if let Some(vid) = version_id {
write_text_element(&mut writer, "VersionId", vid);
}
writer.write_event(Event::End(BytesEnd::new("Deleted"))).unwrap();
}
}
for (key, code, message) in errors {
writer.write_event(Event::Start(BytesStart::new("Error"))).unwrap();
write_text_element(&mut writer, "Key", key);
write_text_element(&mut writer, "Code", code);
write_text_element(&mut writer, "Message", message);
writer.write_event(Event::End(BytesEnd::new("Error"))).unwrap();
}
writer.write_event(Event::End(BytesEnd::new("DeleteResult"))).unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
pub fn list_multipart_uploads_xml(
bucket: &str,
uploads: &[myfsio_common::types::MultipartUploadInfo],
) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
let start = BytesStart::new("ListMultipartUploadsResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
writer.write_event(Event::Start(start)).unwrap();
write_text_element(&mut writer, "Bucket", bucket);
for upload in uploads {
writer.write_event(Event::Start(BytesStart::new("Upload"))).unwrap();
write_text_element(&mut writer, "Key", &upload.key);
write_text_element(&mut writer, "UploadId", &upload.upload_id);
write_text_element(&mut writer, "Initiated", &upload.initiated.to_rfc3339());
writer.write_event(Event::End(BytesEnd::new("Upload"))).unwrap();
}
writer.write_event(Event::End(BytesEnd::new("ListMultipartUploadsResult"))).unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
pub fn list_parts_xml(
bucket: &str,
key: &str,
upload_id: &str,
parts: &[myfsio_common::types::PartMeta],
) -> String {
let mut writer = Writer::new(Cursor::new(Vec::new()));
writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap();
let start = BytesStart::new("ListPartsResult")
.with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]);
writer.write_event(Event::Start(start)).unwrap();
write_text_element(&mut writer, "Bucket", bucket);
write_text_element(&mut writer, "Key", key);
write_text_element(&mut writer, "UploadId", upload_id);
for part in parts {
writer.write_event(Event::Start(BytesStart::new("Part"))).unwrap();
write_text_element(&mut writer, "PartNumber", &part.part_number.to_string());
write_text_element(&mut writer, "ETag", &format!("\"{}\"", part.etag));
write_text_element(&mut writer, "Size", &part.size.to_string());
if let Some(ref lm) = part.last_modified {
write_text_element(&mut writer, "LastModified", &lm.to_rfc3339());
}
writer.write_event(Event::End(BytesEnd::new("Part"))).unwrap();
}
writer.write_event(Event::End(BytesEnd::new("ListPartsResult"))).unwrap();
String::from_utf8(writer.into_inner().into_inner()).unwrap()
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::Utc;
#[test]
fn test_list_buckets_xml() {
let buckets = vec![BucketMeta {
name: "test-bucket".to_string(),
creation_date: Utc::now(),
}];
let xml = list_buckets_xml("owner-id", "owner-name", &buckets);
assert!(xml.contains("<Name>test-bucket</Name>"));
assert!(xml.contains("<ID>owner-id</ID>"));
assert!(xml.contains("ListAllMyBucketsResult"));
}
#[test]
fn test_list_objects_v2_xml() {
let objects = vec![ObjectMeta::new("file.txt".to_string(), 1024, Utc::now())];
let xml = list_objects_v2_xml(
"my-bucket", "", "/", 1000, &objects, &[], false, None, None, 1,
);
assert!(xml.contains("<Key>file.txt</Key>"));
assert!(xml.contains("<Size>1024</Size>"));
assert!(xml.contains("<IsTruncated>false</IsTruncated>"));
}
}