From 4c30efd8025eb4c070d892c13c98b094588047f0 Mon Sep 17 00:00:00 2001 From: kqjy Date: Thu, 2 Apr 2026 21:57:16 +0800 Subject: [PATCH] Update myfsio rust engines - added more implementations --- myfsio-engine/Cargo.lock | 1685 ++++++++++++++++- myfsio-engine/Cargo.toml | 2 + myfsio-engine/crates/myfsio-auth/src/iam.rs | 137 ++ .../crates/myfsio-common/src/types.rs | 4 + myfsio-engine/crates/myfsio-server/Cargo.toml | 4 + .../myfsio-server/src/handlers/config.rs | 284 +++ .../crates/myfsio-server/src/handlers/mod.rs | 476 ++++- .../myfsio-server/src/handlers/select.rs | 552 ++++++ .../crates/myfsio-server/src/main.rs | 25 +- .../myfsio-server/src/middleware/auth.rs | 191 +- .../crates/myfsio-server/tests/integration.rs | 823 +++++++- .../crates/myfsio-xml/src/response.rs | 89 + 12 files changed, 4154 insertions(+), 118 deletions(-) create mode 100644 myfsio-engine/crates/myfsio-server/src/handlers/select.rs diff --git a/myfsio-engine/Cargo.lock b/myfsio-engine/Cargo.lock index 596acf6..f6291fa 100644 --- a/myfsio-engine/Cargo.lock +++ b/myfsio-engine/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aead" version = "0.5.2" @@ -37,6 +43,31 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "version_check", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -67,6 +98,193 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "arrow" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" +dependencies = [ + "ahash 0.8.12", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.16.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ord" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" +dependencies = [ + "bitflags", +] + +[[package]] +name = "arrow-select" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" +dependencies = [ + "ahash 0.8.12", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -143,6 +361,18 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -161,18 +391,70 @@ dependencies = [ "generic-array", ] +[[package]] +name = "borsh" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "bumpalo" version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cbc" version = "0.1.2" @@ -189,6 +471,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -198,6 +482,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.44" @@ -222,6 +512,36 @@ dependencies = [ "inout", ] +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -237,12 +557,27 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -250,7 +585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", - "rand_core", + "rand_core 0.6.4", "typenum", ] @@ -277,6 +612,17 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "digest" version = "0.10.7" @@ -288,6 +634,34 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "duckdb" +version = "1.10501.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f13bc6d6487032fc2825a62ef8b4924b2378a2eb3166e132e5f3141ae9dd633f" +dependencies = [ + "arrow", + "cast", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libduckdb-sys", + "num-integer", + "rust_decimal", + "strum", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -301,21 +675,55 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "foldhash" version = "0.1.5" @@ -331,6 +739,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.32" @@ -387,7 +801,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -436,8 +850,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", ] [[package]] @@ -448,7 +878,7 @@ checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", "wasip2", "wasip3", ] @@ -463,6 +893,27 @@ dependencies = [ "polyval", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -486,6 +937,15 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "heck" version = "0.5.0" @@ -579,6 +1039,24 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", ] [[package]] @@ -587,13 +1065,21 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ + "base64", "bytes", + "futures-channel", + "futures-util", "http", "http-body", "hyper", + "ipnet", + "libc", + "percent-encoding", "pin-project-lite", + "socket2", "tokio", "tower-service", + "tracing", ] [[package]] @@ -620,12 +1106,115 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "id-arena" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.13.0" @@ -648,18 +1237,46 @@ dependencies = [ "generic-array", ] +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iri-string" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.94" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -676,18 +1293,116 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + [[package]] name = "libc" version = "0.2.184" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +[[package]] +name = "libduckdb-sys" +version = "1.10501.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12096c1694924782b3fe21e790630b77bacb4fcb7ad9d7ee0fec626f985bf248" +dependencies = [ + "cc", + "flate2", + "pkg-config", + "reqwest", + "serde", + "serde_json", + "tar", + "vcpkg", + "zip", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" +dependencies = [ + "bitflags", + "libc", + "plain", + "redox_syscall 0.7.3", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "lock_api" version = "0.4.14" @@ -712,6 +1427,12 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "matchit" version = "0.8.4" @@ -740,6 +1461,26 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.0" @@ -748,7 +1489,7 @@ checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -796,7 +1537,7 @@ dependencies = [ "hkdf", "md-5", "myfsio-common", - "rand", + "rand 0.8.5", "serde", "serde_json", "sha2", @@ -814,9 +1555,12 @@ dependencies = [ "base64", "bytes", "chrono", + "crc32fast", + "duckdb", "futures", "http-body-util", "hyper", + "mime_guess", "myfsio-auth", "myfsio-common", "myfsio-crypto", @@ -824,6 +1568,7 @@ dependencies = [ "myfsio-xml", "percent-encoding", "quick-xml", + "roxmltree", "serde", "serde_json", "tempfile", @@ -875,7 +1620,35 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", ] [[package]] @@ -885,6 +1658,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -917,7 +1691,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link", ] @@ -944,6 +1718,18 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "polyval" version = "0.6.2" @@ -956,6 +1742,15 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -972,7 +1767,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", ] [[package]] @@ -984,6 +1788,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "quick-xml" version = "0.37.5" @@ -994,6 +1818,61 @@ dependencies = [ "serde", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.2", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + [[package]] name = "quote" version = "1.0.45" @@ -1003,12 +1882,24 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.5" @@ -1016,8 +1907,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", ] [[package]] @@ -1027,7 +1928,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -1039,6 +1950,15 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1048,6 +1968,15 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_syscall" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.12.3" @@ -1077,6 +2006,127 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rkyv" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "roxmltree" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" + +[[package]] +name = "rust_decimal" +version = "1.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ce901f9a19d251159075a4c37af514c3b8ef99c22e02dd8c19161cf397ee94a" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand 0.8.5", + "rkyv", + "serde", + "serde_json", + "wasm-bindgen", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + [[package]] name = "rustix" version = "1.1.4" @@ -1087,7 +2137,42 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -1108,6 +2193,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "semver" version = "1.0.27" @@ -1141,7 +2232,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1216,6 +2307,18 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "slab" version = "0.4.12" @@ -1235,7 +2338,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] @@ -1244,6 +2374,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.117" @@ -1260,6 +2401,37 @@ name = "sync_wrapper" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tar" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +dependencies = [ + "filetime", + "libc", + "xattr", +] [[package]] name = "tempfile" @@ -1271,7 +2443,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1291,7 +2463,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1303,6 +2475,25 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.11.0" @@ -1332,7 +2523,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1343,7 +2534,17 @@ checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", ] [[package]] @@ -1359,6 +2560,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.10+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a82418ca169e235e6c399a84e395ab6debeb3bc90edc959bf0f48647c6a32d1b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + [[package]] name = "tower" version = "0.5.3" @@ -1383,9 +2614,12 @@ checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "bitflags", "bytes", + "futures-util", "http", "http-body", + "iri-string", "pin-project-lite", + "tower", "tower-layer", "tower-service", "tracing", @@ -1423,7 +2657,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1461,12 +2695,24 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1482,6 +2728,18 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -1498,6 +2756,30 @@ dependencies = [ "subtle", ] +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "uuid" version = "1.23.0" @@ -1515,12 +2797,27 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1554,10 +2851,21 @@ dependencies = [ "cfg-if", "once_cell", "rustversion", + "serde", "wasm-bindgen-macro", "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.117" @@ -1577,7 +2885,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -1624,6 +2932,35 @@ dependencies = [ "semver", ] +[[package]] +name = "web-sys" +version = "0.3.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -1645,7 +2982,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1656,7 +2993,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1683,6 +3020,24 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1692,6 +3047,144 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -1722,7 +3215,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -1738,7 +3231,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -1780,6 +3273,54 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.48" @@ -1797,11 +3338,103 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] +[[package]] +name = "zerofrom" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zip" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap", + "memchr", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/myfsio-engine/Cargo.toml b/myfsio-engine/Cargo.toml index 45a02a8..405e799 100644 --- a/myfsio-engine/Cargo.toml +++ b/myfsio-engine/Cargo.toml @@ -41,3 +41,5 @@ base64 = "0.22" tokio-util = { version = "0.7", features = ["io"] } futures = "0.3" dashmap = "6" +crc32fast = "1" +duckdb = { version = "1", features = ["bundled"] } diff --git a/myfsio-engine/crates/myfsio-auth/src/iam.rs b/myfsio-engine/crates/myfsio-auth/src/iam.rs index 0a9071d..18a7e51 100644 --- a/myfsio-engine/crates/myfsio-auth/src/iam.rs +++ b/myfsio-engine/crates/myfsio-auth/src/iam.rs @@ -274,6 +274,61 @@ impl IamService { self.get_principal(access_key) } + pub fn authorize( + &self, + principal: &Principal, + bucket_name: Option<&str>, + action: &str, + object_key: Option<&str>, + ) -> bool { + self.reload_if_needed(); + + if principal.is_admin { + return true; + } + + let normalized_bucket = bucket_name + .unwrap_or("*") + .trim() + .to_ascii_lowercase(); + let normalized_action = action.trim().to_ascii_lowercase(); + + let state = self.state.read(); + let user = match state.user_records.get(&principal.user_id) { + Some(u) => u, + None => return false, + }; + + if !user.enabled { + return false; + } + + if let Some(ref expires_at) = user.expires_at { + if let Ok(exp) = expires_at.parse::>() { + if Utc::now() > exp { + return false; + } + } + } + + for policy in &user.policies { + if !bucket_matches(&policy.bucket, &normalized_bucket) { + continue; + } + if !action_matches(&policy.actions, &normalized_action) { + continue; + } + if let Some(key) = object_key { + if !prefix_matches(&policy.prefix, key) { + continue; + } + } + return true; + } + + false + } + pub async fn list_users(&self) -> Vec { self.reload_if_needed(); let state = self.state.read(); @@ -353,6 +408,33 @@ impl IamService { } } +fn bucket_matches(policy_bucket: &str, bucket: &str) -> bool { + let pb = policy_bucket.trim().to_ascii_lowercase(); + pb == "*" || pb == bucket +} + +fn action_matches(policy_actions: &[String], action: &str) -> bool { + for policy_action in policy_actions { + let pa = policy_action.trim().to_ascii_lowercase(); + if pa == "*" || pa == action { + return true; + } + if pa == "iam:*" && action.starts_with("iam:") { + return true; + } + } + false +} + +fn prefix_matches(policy_prefix: &str, object_key: &str) -> bool { + let p = policy_prefix.trim(); + if p.is_empty() || p == "*" { + return true; + } + let base = p.trim_end_matches('*'); + object_key.starts_with(base) +} + #[cfg(test)] mod tests { use super::*; @@ -496,4 +578,59 @@ mod tests { let svc = IamService::new(tmp.path().to_path_buf()); assert!(svc.get_secret_key("INACTIVE_KEY").is_none()); } + + #[test] + fn test_authorize_allows_matching_policy() { + let json = serde_json::json!({ + "version": 2, + "users": [{ + "user_id": "u-reader", + "display_name": "reader", + "enabled": true, + "access_keys": [{ + "access_key": "READER_KEY", + "secret_key": "reader-secret", + "status": "active" + }], + "policies": [{ + "bucket": "docs", + "actions": ["read"], + "prefix": "reports/" + }] + }] + }) + .to_string(); + + let mut tmp = tempfile::NamedTempFile::new().unwrap(); + tmp.write_all(json.as_bytes()).unwrap(); + tmp.flush().unwrap(); + + let svc = IamService::new(tmp.path().to_path_buf()); + let principal = svc.get_principal("READER_KEY").unwrap(); + + assert!(svc.authorize( + &principal, + Some("docs"), + "read", + Some("reports/2026.csv"), + )); + assert!(!svc.authorize( + &principal, + Some("docs"), + "write", + Some("reports/2026.csv"), + )); + assert!(!svc.authorize( + &principal, + Some("docs"), + "read", + Some("private/2026.csv"), + )); + assert!(!svc.authorize( + &principal, + Some("other"), + "read", + Some("reports/2026.csv"), + )); + } } diff --git a/myfsio-engine/crates/myfsio-common/src/types.rs b/myfsio-engine/crates/myfsio-common/src/types.rs index 73f6aee..a07d565 100644 --- a/myfsio-engine/crates/myfsio-common/src/types.rs +++ b/myfsio-engine/crates/myfsio-common/src/types.rs @@ -144,6 +144,10 @@ pub struct BucketConfig { pub logging: Option, #[serde(default)] pub object_lock: Option, + #[serde(default)] + pub policy: Option, + #[serde(default)] + pub replication: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/myfsio-engine/crates/myfsio-server/Cargo.toml b/myfsio-engine/crates/myfsio-server/Cargo.toml index 7f1f9df..78ab4ea 100644 --- a/myfsio-engine/crates/myfsio-server/Cargo.toml +++ b/myfsio-engine/crates/myfsio-server/Cargo.toml @@ -27,6 +27,10 @@ futures = { workspace = true } http-body-util = "0.1" percent-encoding = { workspace = true } quick-xml = { workspace = true } +mime_guess = "2" +crc32fast = { workspace = true } +duckdb = { workspace = true } +roxmltree = "0.20" [dev-dependencies] tempfile = "3" diff --git a/myfsio-engine/crates/myfsio-server/src/handlers/config.rs b/myfsio-engine/crates/myfsio-server/src/handlers/config.rs index 01dac09..6b95c3e 100644 --- a/myfsio-engine/crates/myfsio-server/src/handlers/config.rs +++ b/myfsio-engine/crates/myfsio-server/src/handlers/config.rs @@ -17,6 +17,15 @@ fn storage_err(err: myfsio_storage::error::StorageError) -> Response { (status, [("content-type", "application/xml")], s3err.to_xml()).into_response() } +fn json_response(status: StatusCode, value: serde_json::Value) -> Response { + ( + status, + [("content-type", "application/json")], + value.to_string(), + ) + .into_response() +} + pub async fn get_versioning(state: &AppState, bucket: &str) -> Response { match state.storage.is_versioning_enabled(bucket).await { Ok(enabled) => { @@ -271,6 +280,281 @@ pub async fn delete_lifecycle(state: &AppState, bucket: &str) -> Response { } } +pub async fn get_quota(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(config) => { + if let Some(quota) = &config.quota { + let usage = match state.storage.bucket_stats(bucket).await { + Ok(s) => s, + Err(e) => return storage_err(e), + }; + json_response( + StatusCode::OK, + serde_json::json!({ + "quota": { + "max_size_bytes": quota.max_bytes, + "max_objects": quota.max_objects, + }, + "usage": { + "bytes": usage.bytes, + "objects": usage.objects, + } + }), + ) + } else { + xml_response( + StatusCode::NOT_FOUND, + S3Error::new(S3ErrorCode::NoSuchKey, "No quota configuration found").to_xml(), + ) + } + } + Err(e) => storage_err(e), + } +} + +pub async fn put_quota(state: &AppState, bucket: &str, body: Body) -> Response { + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(collected) => collected.to_bytes(), + Err(_) => { + return xml_response( + StatusCode::BAD_REQUEST, + S3Error::new(S3ErrorCode::InvalidArgument, "Invalid quota payload").to_xml(), + ); + } + }; + + let payload: serde_json::Value = match serde_json::from_slice(&body_bytes) { + Ok(v) => v, + Err(_) => { + return xml_response( + StatusCode::BAD_REQUEST, + S3Error::new(S3ErrorCode::InvalidArgument, "Request body must be valid JSON").to_xml(), + ); + } + }; + + let max_size = payload + .get("max_size_bytes") + .and_then(|v| v.as_u64()); + let max_objects = payload + .get("max_objects") + .and_then(|v| v.as_u64()); + + if max_size.is_none() && max_objects.is_none() { + return xml_response( + StatusCode::BAD_REQUEST, + S3Error::new( + S3ErrorCode::InvalidArgument, + "At least one of max_size_bytes or max_objects is required", + ) + .to_xml(), + ); + } + + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.quota = Some(myfsio_common::types::QuotaConfig { + max_bytes: max_size, + max_objects, + }); + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::OK.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn delete_quota(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.quota = None; + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn get_policy(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(config) => { + if let Some(policy) = &config.policy { + json_response(StatusCode::OK, policy.clone()) + } else { + xml_response( + StatusCode::NOT_FOUND, + S3Error::new(S3ErrorCode::NoSuchKey, "No bucket policy attached").to_xml(), + ) + } + } + Err(e) => storage_err(e), + } +} + +pub async fn put_policy(state: &AppState, bucket: &str, body: Body) -> Response { + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(collected) => collected.to_bytes(), + Err(_) => { + return xml_response( + StatusCode::BAD_REQUEST, + S3Error::new(S3ErrorCode::MalformedXML, "Failed to read policy body").to_xml(), + ); + } + }; + + let policy: serde_json::Value = match serde_json::from_slice(&body_bytes) { + Ok(v) => v, + Err(_) => { + return xml_response( + StatusCode::BAD_REQUEST, + S3Error::new(S3ErrorCode::InvalidArgument, "Policy document must be JSON").to_xml(), + ); + } + }; + + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.policy = Some(policy); + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn delete_policy(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.policy = None; + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn get_policy_status(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(config) => { + let is_public = config + .policy + .as_ref() + .map(policy_is_public) + .unwrap_or(false); + let xml = format!( + "{}", + if is_public { "TRUE" } else { "FALSE" } + ); + xml_response(StatusCode::OK, xml) + } + Err(e) => storage_err(e), + } +} + +pub async fn get_replication(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(config) => { + if let Some(replication) = &config.replication { + match replication { + serde_json::Value::String(s) => xml_response(StatusCode::OK, s.clone()), + other => xml_response(StatusCode::OK, other.to_string()), + } + } else { + xml_response( + StatusCode::NOT_FOUND, + S3Error::new( + S3ErrorCode::NoSuchKey, + "Replication configuration not found", + ) + .to_xml(), + ) + } + } + Err(e) => storage_err(e), + } +} + +pub async fn put_replication(state: &AppState, bucket: &str, body: Body) -> Response { + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(collected) => collected.to_bytes(), + Err(_) => { + return xml_response( + StatusCode::BAD_REQUEST, + S3Error::new(S3ErrorCode::MalformedXML, "Failed to read replication body").to_xml(), + ); + } + }; + + if body_bytes.is_empty() { + return xml_response( + StatusCode::BAD_REQUEST, + S3Error::new(S3ErrorCode::MalformedXML, "Request body is required").to_xml(), + ); + } + + let body_str = String::from_utf8_lossy(&body_bytes).to_string(); + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.replication = Some(serde_json::Value::String(body_str)); + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::OK.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn delete_replication(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.replication = None; + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +fn policy_is_public(policy: &serde_json::Value) -> bool { + let statements = match policy.get("Statement") { + Some(serde_json::Value::Array(items)) => items, + Some(item) => { + return is_allow_public_statement(item); + } + None => return false, + }; + + statements.iter().any(is_allow_public_statement) +} + +fn is_allow_public_statement(statement: &serde_json::Value) -> bool { + let effect_allow = statement + .get("Effect") + .and_then(|v| v.as_str()) + .map(|s| s.eq_ignore_ascii_case("allow")) + .unwrap_or(false); + if !effect_allow { + return false; + } + + match statement.get("Principal") { + Some(serde_json::Value::String(s)) => s == "*", + Some(serde_json::Value::Object(obj)) => obj.values().any(|v| v == "*"), + _ => false, + } +} + pub async fn get_acl(state: &AppState, bucket: &str) -> Response { match state.storage.get_bucket_config(bucket).await { Ok(config) => { diff --git a/myfsio-engine/crates/myfsio-server/src/handlers/mod.rs b/myfsio-engine/crates/myfsio-server/src/handlers/mod.rs index 9055a0a..c6ce18f 100644 --- a/myfsio-engine/crates/myfsio-server/src/handlers/mod.rs +++ b/myfsio-engine/crates/myfsio-server/src/handlers/mod.rs @@ -1,5 +1,6 @@ mod config; pub mod kms; +mod select; use std::collections::HashMap; @@ -7,8 +8,11 @@ use axum::body::Body; use axum::extract::{Path, Query, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::{IntoResponse, Response}; +use base64::engine::general_purpose::URL_SAFE; +use base64::Engine; +use chrono::{DateTime, Utc}; -use myfsio_common::error::S3Error; +use myfsio_common::error::{S3Error, S3ErrorCode}; use myfsio_common::types::PartInfo; use myfsio_storage::traits::StorageEngine; use tokio::io::AsyncSeekExt; @@ -18,7 +22,15 @@ use crate::state::AppState; fn s3_error_response(err: S3Error) -> Response { let status = StatusCode::from_u16(err.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); - let body = err.to_xml(); + let resource = if err.resource.is_empty() { + "/".to_string() + } else { + err.resource.clone() + }; + let body = err + .with_resource(resource) + .with_request_id(uuid::Uuid::new_v4().simple().to_string()) + .to_xml(); ( status, [("content-type", "application/xml")], @@ -52,6 +64,9 @@ pub async fn create_bucket( Query(query): Query, body: Body, ) -> Response { + if query.quota.is_some() { + return config::put_quota(&state, &bucket, body).await; + } if query.versioning.is_some() { return config::put_versioning(&state, &bucket, body).await; } @@ -70,6 +85,12 @@ pub async fn create_bucket( if query.acl.is_some() { return config::put_acl(&state, &bucket, body).await; } + if query.policy.is_some() { + return config::put_policy(&state, &bucket, body).await; + } + if query.replication.is_some() { + return config::put_replication(&state, &bucket, body).await; + } if query.website.is_some() { return config::put_website(&state, &bucket, body).await; } @@ -91,6 +112,7 @@ pub async fn create_bucket( pub struct BucketQuery { #[serde(rename = "list-type")] pub list_type: Option, + pub marker: Option, pub prefix: Option, pub delimiter: Option, #[serde(rename = "max-keys")] @@ -108,7 +130,11 @@ pub struct BucketQuery { pub encryption: Option, pub lifecycle: Option, pub acl: Option, + pub quota: Option, pub policy: Option, + #[serde(rename = "policyStatus")] + pub policy_status: Option, + pub replication: Option, pub website: Option, #[serde(rename = "object-lock")] pub object_lock: Option, @@ -128,6 +154,9 @@ pub async fn get_bucket( ); } + if query.quota.is_some() { + return config::get_quota(&state, &bucket).await; + } if query.versioning.is_some() { return config::get_versioning(&state, &bucket).await; } @@ -149,6 +178,15 @@ pub async fn get_bucket( if query.acl.is_some() { return config::get_acl(&state, &bucket).await; } + if query.policy.is_some() { + return config::get_policy(&state, &bucket).await; + } + if query.policy_status.is_some() { + return config::get_policy_status(&state, &bucket).await; + } + if query.replication.is_some() { + return config::get_replication(&state, &bucket).await; + } if query.website.is_some() { return config::get_website(&state, &bucket).await; } @@ -171,28 +209,80 @@ pub async fn get_bucket( let prefix = query.prefix.clone().unwrap_or_default(); let delimiter = query.delimiter.clone().unwrap_or_default(); let max_keys = query.max_keys.unwrap_or(1000); + let marker = query.marker.clone().unwrap_or_default(); + let list_type = query.list_type.clone().unwrap_or_default(); + let is_v2 = list_type == "2"; + + let effective_start = if is_v2 { + if let Some(token) = query.continuation_token.as_deref() { + match URL_SAFE.decode(token) { + Ok(bytes) => match String::from_utf8(bytes) { + Ok(decoded) => Some(decoded), + Err(_) => { + return s3_error_response(S3Error::new( + S3ErrorCode::InvalidArgument, + "Invalid continuation token", + )); + } + }, + Err(_) => { + return s3_error_response(S3Error::new( + S3ErrorCode::InvalidArgument, + "Invalid continuation token", + )); + } + } + } else { + query.start_after.clone() + } + } else if marker.is_empty() { + None + } else { + Some(marker.clone()) + }; if delimiter.is_empty() { let params = myfsio_common::types::ListParams { max_keys, - continuation_token: query.continuation_token.clone(), + continuation_token: effective_start.clone(), prefix: if prefix.is_empty() { None } else { Some(prefix.clone()) }, - start_after: query.start_after.clone(), + start_after: if is_v2 { query.start_after.clone() } else { None }, }; match state.storage.list_objects(&bucket, ¶ms).await { Ok(result) => { - let xml = myfsio_xml::response::list_objects_v2_xml( - &bucket, - &prefix, - &delimiter, - max_keys, - &result.objects, - &[], - result.is_truncated, - query.continuation_token.as_deref(), - result.next_continuation_token.as_deref(), - result.objects.len(), - ); + let next_marker = result + .next_continuation_token + .clone() + .or_else(|| result.objects.last().map(|o| o.key.clone())); + let xml = if is_v2 { + let next_token = next_marker + .as_deref() + .map(|s| URL_SAFE.encode(s.as_bytes())); + myfsio_xml::response::list_objects_v2_xml( + &bucket, + &prefix, + &delimiter, + max_keys, + &result.objects, + &[], + result.is_truncated, + query.continuation_token.as_deref(), + next_token.as_deref(), + result.objects.len(), + ) + } else { + myfsio_xml::response::list_objects_v1_xml( + &bucket, + &prefix, + &marker, + &delimiter, + max_keys, + &result.objects, + &[], + result.is_truncated, + next_marker.as_deref(), + ) + }; (StatusCode::OK, [("content-type", "application/xml")], xml).into_response() } Err(e) => storage_err_response(e), @@ -202,22 +292,40 @@ pub async fn get_bucket( prefix, delimiter: delimiter.clone(), max_keys, - continuation_token: query.continuation_token.clone(), + continuation_token: effective_start, }; match state.storage.list_objects_shallow(&bucket, ¶ms).await { Ok(result) => { - let xml = myfsio_xml::response::list_objects_v2_xml( - &bucket, - ¶ms.prefix, - &delimiter, - max_keys, - &result.objects, - &result.common_prefixes, - result.is_truncated, - query.continuation_token.as_deref(), - result.next_continuation_token.as_deref(), - result.objects.len() + result.common_prefixes.len(), - ); + let xml = if is_v2 { + let next_token = result + .next_continuation_token + .as_deref() + .map(|s| URL_SAFE.encode(s.as_bytes())); + myfsio_xml::response::list_objects_v2_xml( + &bucket, + ¶ms.prefix, + &delimiter, + max_keys, + &result.objects, + &result.common_prefixes, + result.is_truncated, + query.continuation_token.as_deref(), + next_token.as_deref(), + result.objects.len() + result.common_prefixes.len(), + ) + } else { + myfsio_xml::response::list_objects_v1_xml( + &bucket, + ¶ms.prefix, + &marker, + &delimiter, + max_keys, + &result.objects, + &result.common_prefixes, + result.is_truncated, + result.next_continuation_token.as_deref(), + ) + }; (StatusCode::OK, [("content-type", "application/xml")], xml).into_response() } Err(e) => storage_err_response(e), @@ -243,6 +351,9 @@ pub async fn delete_bucket( Path(bucket): Path, Query(query): Query, ) -> Response { + if query.quota.is_some() { + return config::delete_quota(&state, &bucket).await; + } if query.tagging.is_some() { return config::delete_tagging(&state, &bucket).await; } @@ -258,6 +369,12 @@ pub async fn delete_bucket( if query.website.is_some() { return config::delete_website(&state, &bucket).await; } + if query.policy.is_some() { + return config::delete_policy(&state, &bucket).await; + } + if query.replication.is_some() { + return config::delete_replication(&state, &bucket).await; + } match state.storage.delete_bucket(&bucket).await { Ok(()) => StatusCode::NO_CONTENT.into_response(), @@ -285,6 +402,8 @@ pub async fn head_bucket( #[derive(serde::Deserialize, Default)] pub struct ObjectQuery { pub uploads: Option, + pub attributes: Option, + pub select: Option, #[serde(rename = "uploadId")] pub upload_id: Option, #[serde(rename = "partNumber")] @@ -329,6 +448,27 @@ fn apply_response_overrides(headers: &mut HeaderMap, query: &ObjectQuery) { } } +fn guessed_content_type(key: &str, explicit: Option<&str>) -> String { + explicit + .filter(|v| !v.trim().is_empty()) + .map(|v| v.to_string()) + .unwrap_or_else(|| { + mime_guess::from_path(key) + .first_raw() + .unwrap_or("application/octet-stream") + .to_string() + }) +} + +fn insert_content_type(headers: &mut HeaderMap, key: &str, explicit: Option<&str>) { + let value = guessed_content_type(key, explicit); + if let Ok(header_value) = value.parse() { + headers.insert("content-type", header_value); + } else { + headers.insert("content-type", "application/octet-stream".parse().unwrap()); + } +} + pub async fn put_object( State(state): State, Path((bucket, key)): Path<(String, String)>, @@ -356,16 +496,18 @@ pub async fn put_object( } if let Some(copy_source) = headers.get("x-amz-copy-source").and_then(|v| v.to_str().ok()) { - return copy_object_handler(&state, copy_source, &bucket, &key).await; + return copy_object_handler(&state, copy_source, &bucket, &key, &headers).await; } - let content_type = headers - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); + let content_type = guessed_content_type( + &key, + headers + .get("content-type") + .and_then(|v| v.to_str().ok()), + ); let mut metadata = HashMap::new(); - metadata.insert("__content_type__".to_string(), content_type.to_string()); + metadata.insert("__content_type__".to_string(), content_type); for (name, value) in headers.iter() { let name_str = name.as_str(); @@ -460,6 +602,20 @@ pub async fn get_object( if query.legal_hold.is_some() { return config::get_object_legal_hold(&state, &bucket, &key).await; } + if query.attributes.is_some() { + return object_attributes_handler(&state, &bucket, &key, &headers).await; + } + if let Some(ref upload_id) = query.upload_id { + return list_parts_handler(&state, &bucket, &key, upload_id).await; + } + + let head_meta = match state.storage.head_object(&bucket, &key).await { + Ok(m) => m, + Err(e) => return storage_err_response(e), + }; + if let Some(resp) = evaluate_get_preconditions(&headers, &head_meta) { + return resp; + } let range_header = headers .get("range") @@ -504,13 +660,7 @@ pub async fn get_object( let stream = ReaderStream::new(file); let body = Body::from_stream(stream); - let meta = match state.storage.head_object(&bucket, &key).await { - Ok(m) => m, - Err(e) => { - let _ = tokio::fs::remove_file(&dec_tmp).await; - return storage_err_response(e); - } - }; + let meta = head_meta.clone(); let tmp_path = dec_tmp.clone(); tokio::spawn(async move { @@ -523,11 +673,7 @@ pub async fn get_object( if let Some(ref etag) = meta.etag { resp_headers.insert("etag", format!("\"{}\"", etag).parse().unwrap()); } - if let Some(ref ct) = meta.content_type { - resp_headers.insert("content-type", ct.parse().unwrap()); - } else { - resp_headers.insert("content-type", "application/octet-stream".parse().unwrap()); - } + insert_content_type(&mut resp_headers, &key, meta.content_type.as_deref()); resp_headers.insert( "last-modified", meta.last_modified.format("%a, %d %b %Y %H:%M:%S GMT").to_string().parse().unwrap(), @@ -559,11 +705,7 @@ pub async fn get_object( if let Some(ref etag) = meta.etag { headers.insert("etag", format!("\"{}\"", etag).parse().unwrap()); } - if let Some(ref ct) = meta.content_type { - headers.insert("content-type", ct.parse().unwrap()); - } else { - headers.insert("content-type", "application/octet-stream".parse().unwrap()); - } + insert_content_type(&mut headers, &key, meta.content_type.as_deref()); headers.insert( "last-modified", meta.last_modified @@ -595,6 +737,7 @@ pub async fn post_object( State(state): State, Path((bucket, key)): Path<(String, String)>, Query(query): Query, + headers: HeaderMap, body: Body, ) -> Response { if query.uploads.is_some() { @@ -605,6 +748,10 @@ pub async fn post_object( return complete_multipart_handler(&state, &bucket, &key, upload_id, body).await; } + if query.select.is_some() { + return select::post_select_object_content(&state, &bucket, &key, &headers, body).await; + } + (StatusCode::METHOD_NOT_ALLOWED).into_response() } @@ -630,19 +777,19 @@ pub async fn delete_object( pub async fn head_object( State(state): State, Path((bucket, key)): Path<(String, String)>, + headers: HeaderMap, ) -> Response { match state.storage.head_object(&bucket, &key).await { Ok(meta) => { + if let Some(resp) = evaluate_get_preconditions(&headers, &meta) { + return resp; + } let mut headers = HeaderMap::new(); headers.insert("content-length", meta.size.to_string().parse().unwrap()); if let Some(ref etag) = meta.etag { headers.insert("etag", format!("\"{}\"", etag).parse().unwrap()); } - if let Some(ref ct) = meta.content_type { - headers.insert("content-type", ct.parse().unwrap()); - } else { - headers.insert("content-type", "application/octet-stream".parse().unwrap()); - } + insert_content_type(&mut headers, &key, meta.content_type.as_deref()); headers.insert( "last-modified", meta.last_modified @@ -782,11 +929,80 @@ async fn list_multipart_uploads_handler( } } +async fn list_parts_handler( + state: &AppState, + bucket: &str, + key: &str, + upload_id: &str, +) -> Response { + match state.storage.list_parts(bucket, upload_id).await { + Ok(parts) => { + let xml = myfsio_xml::response::list_parts_xml(bucket, key, upload_id, &parts); + (StatusCode::OK, [("content-type", "application/xml")], xml).into_response() + } + Err(e) => storage_err_response(e), + } +} + +async fn object_attributes_handler( + state: &AppState, + bucket: &str, + key: &str, + headers: &HeaderMap, +) -> Response { + let meta = match state.storage.head_object(bucket, key).await { + Ok(m) => m, + Err(e) => return storage_err_response(e), + }; + + let requested = headers + .get("x-amz-object-attributes") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + let attrs: std::collections::HashSet = requested + .split(',') + .map(|s| s.trim().to_ascii_lowercase()) + .filter(|s| !s.is_empty()) + .collect(); + let all = attrs.is_empty(); + + let mut xml = String::from( + "" + ); + xml.push_str(""); + + if all || attrs.contains("etag") { + if let Some(etag) = &meta.etag { + xml.push_str(&format!("{}", xml_escape(etag))); + } + } + if all || attrs.contains("storageclass") { + let sc = meta + .storage_class + .as_deref() + .unwrap_or("STANDARD"); + xml.push_str(&format!("{}", xml_escape(sc))); + } + if all || attrs.contains("objectsize") { + xml.push_str(&format!("{}", meta.size)); + } + if attrs.contains("checksum") { + xml.push_str(""); + } + if attrs.contains("objectparts") { + xml.push_str(""); + } + + xml.push_str(""); + (StatusCode::OK, [("content-type", "application/xml")], xml).into_response() +} + async fn copy_object_handler( state: &AppState, copy_source: &str, dst_bucket: &str, dst_key: &str, + headers: &HeaderMap, ) -> Response { let source = copy_source.strip_prefix('/').unwrap_or(copy_source); let (src_bucket, src_key) = match source.split_once('/') { @@ -799,6 +1015,14 @@ async fn copy_object_handler( } }; + let source_meta = match state.storage.head_object(src_bucket, src_key).await { + Ok(m) => m, + Err(e) => return storage_err_response(e), + }; + if let Some(resp) = evaluate_copy_preconditions(headers, &source_meta) { + return resp; + } + match state.storage.copy_object(src_bucket, src_key, dst_bucket, dst_key).await { Ok(meta) => { let etag = meta.etag.as_deref().unwrap_or(""); @@ -908,11 +1132,7 @@ async fn range_get_handler( if let Some(ref etag) = meta.etag { headers.insert("etag", format!("\"{}\"", etag).parse().unwrap()); } - if let Some(ref ct) = meta.content_type { - headers.insert("content-type", ct.parse().unwrap()); - } else { - headers.insert("content-type", "application/octet-stream".parse().unwrap()); - } + insert_content_type(&mut headers, key, meta.content_type.as_deref()); headers.insert("accept-ranges", "bytes".parse().unwrap()); apply_response_overrides(&mut headers, query); @@ -920,6 +1140,138 @@ async fn range_get_handler( (StatusCode::PARTIAL_CONTENT, headers, body).into_response() } +fn evaluate_get_preconditions( + headers: &HeaderMap, + meta: &myfsio_common::types::ObjectMeta, +) -> Option { + if let Some(value) = headers.get("if-match").and_then(|v| v.to_str().ok()) { + if !etag_condition_matches(value, meta.etag.as_deref()) { + return Some(s3_error_response(S3Error::from_code( + S3ErrorCode::PreconditionFailed, + ))); + } + } + + if let Some(value) = headers + .get("if-unmodified-since") + .and_then(|v| v.to_str().ok()) + { + if let Some(t) = parse_http_date(value) { + if meta.last_modified > t { + return Some(s3_error_response(S3Error::from_code( + S3ErrorCode::PreconditionFailed, + ))); + } + } + } + + if let Some(value) = headers.get("if-none-match").and_then(|v| v.to_str().ok()) { + if etag_condition_matches(value, meta.etag.as_deref()) { + return Some(StatusCode::NOT_MODIFIED.into_response()); + } + } + + if let Some(value) = headers + .get("if-modified-since") + .and_then(|v| v.to_str().ok()) + { + if let Some(t) = parse_http_date(value) { + if meta.last_modified <= t { + return Some(StatusCode::NOT_MODIFIED.into_response()); + } + } + } + + None +} + +fn evaluate_copy_preconditions( + headers: &HeaderMap, + source_meta: &myfsio_common::types::ObjectMeta, +) -> Option { + if let Some(value) = headers + .get("x-amz-copy-source-if-match") + .and_then(|v| v.to_str().ok()) + { + if !etag_condition_matches(value, source_meta.etag.as_deref()) { + return Some(s3_error_response(S3Error::from_code( + S3ErrorCode::PreconditionFailed, + ))); + } + } + + if let Some(value) = headers + .get("x-amz-copy-source-if-none-match") + .and_then(|v| v.to_str().ok()) + { + if etag_condition_matches(value, source_meta.etag.as_deref()) { + return Some(s3_error_response(S3Error::from_code( + S3ErrorCode::PreconditionFailed, + ))); + } + } + + if let Some(value) = headers + .get("x-amz-copy-source-if-modified-since") + .and_then(|v| v.to_str().ok()) + { + if let Some(t) = parse_http_date(value) { + if source_meta.last_modified <= t { + return Some(s3_error_response(S3Error::from_code( + S3ErrorCode::PreconditionFailed, + ))); + } + } + } + + if let Some(value) = headers + .get("x-amz-copy-source-if-unmodified-since") + .and_then(|v| v.to_str().ok()) + { + if let Some(t) = parse_http_date(value) { + if source_meta.last_modified > t { + return Some(s3_error_response(S3Error::from_code( + S3ErrorCode::PreconditionFailed, + ))); + } + } + } + + None +} + +fn parse_http_date(value: &str) -> Option> { + DateTime::parse_from_rfc2822(value) + .ok() + .map(|dt| dt.with_timezone(&Utc)) +} + +fn etag_condition_matches(condition: &str, etag: Option<&str>) -> bool { + let trimmed = condition.trim(); + if trimmed == "*" { + return true; + } + + let current = match etag { + Some(e) => e.trim_matches('"'), + None => return false, + }; + + trimmed + .split(',') + .map(|v| v.trim().trim_matches('"')) + .any(|candidate| candidate == current || candidate == "*") +} + +fn xml_escape(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + fn parse_range(range_str: &str, total_size: u64) -> Option<(u64, u64)> { let range_spec = range_str.strip_prefix("bytes=")?; diff --git a/myfsio-engine/crates/myfsio-server/src/handlers/select.rs b/myfsio-engine/crates/myfsio-server/src/handlers/select.rs new file mode 100644 index 0000000..211f35a --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/src/handlers/select.rs @@ -0,0 +1,552 @@ +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use axum::body::Body; +use axum::http::{HeaderMap, HeaderName, StatusCode}; +use axum::response::{IntoResponse, Response}; +use base64::Engine; +use bytes::Bytes; +use crc32fast::Hasher; +use duckdb::types::ValueRef; +use duckdb::Connection; +use futures::stream; +use http_body_util::BodyExt; +use myfsio_common::error::{S3Error, S3ErrorCode}; +use myfsio_storage::traits::StorageEngine; + +use crate::state::AppState; + +#[cfg(target_os = "windows")] +#[link(name = "Rstrtmgr")] +extern "system" {} + +const CHUNK_SIZE: usize = 65_536; + +pub async fn post_select_object_content( + state: &AppState, + bucket: &str, + key: &str, + headers: &HeaderMap, + body: Body, +) -> Response { + if let Some(resp) = require_xml_content_type(headers) { + return resp; + } + + let body_bytes = match body.collect().await { + Ok(collected) => collected.to_bytes(), + Err(_) => { + return s3_error_response(S3Error::new( + S3ErrorCode::MalformedXML, + "Unable to parse XML document", + )); + } + }; + + let request = match parse_select_request(&body_bytes) { + Ok(r) => r, + Err(err) => return s3_error_response(err), + }; + + let object_path = match state.storage.get_object_path(bucket, key).await { + Ok(path) => path, + Err(_) => { + return s3_error_response(S3Error::new( + S3ErrorCode::NoSuchKey, + "Object not found", + )); + } + }; + + let join_res = tokio::task::spawn_blocking(move || execute_select_query(object_path, request)).await; + let chunks = match join_res { + Ok(Ok(chunks)) => chunks, + Ok(Err(message)) => { + return s3_error_response(S3Error::new(S3ErrorCode::InvalidRequest, message)); + } + Err(_) => { + return s3_error_response(S3Error::new( + S3ErrorCode::InternalError, + "SelectObjectContent execution failed", + )); + } + }; + + let bytes_returned: usize = chunks.iter().map(|c| c.len()).sum(); + let mut events: Vec = Vec::with_capacity(chunks.len() + 2); + for chunk in chunks { + events.push(Bytes::from(encode_select_event("Records", &chunk))); + } + + let stats_payload = build_stats_xml(0, bytes_returned); + events.push(Bytes::from(encode_select_event("Stats", stats_payload.as_bytes()))); + events.push(Bytes::from(encode_select_event("End", b""))); + + let stream = stream::iter(events.into_iter().map(Ok::)); + let body = Body::from_stream(stream); + + let mut response = (StatusCode::OK, body).into_response(); + response.headers_mut().insert( + HeaderName::from_static("content-type"), + "application/octet-stream".parse().unwrap(), + ); + response.headers_mut().insert( + HeaderName::from_static("x-amz-request-charged"), + "requester".parse().unwrap(), + ); + response +} + +#[derive(Clone)] +struct SelectRequest { + expression: String, + input_format: InputFormat, + output_format: OutputFormat, +} + +#[derive(Clone)] +enum InputFormat { + Csv(CsvInputConfig), + Json(JsonInputConfig), + Parquet, +} + +#[derive(Clone)] +struct CsvInputConfig { + file_header_info: String, + field_delimiter: String, + quote_character: String, +} + +#[derive(Clone)] +struct JsonInputConfig { + json_type: String, +} + +#[derive(Clone)] +enum OutputFormat { + Csv(CsvOutputConfig), + Json(JsonOutputConfig), +} + +#[derive(Clone)] +struct CsvOutputConfig { + field_delimiter: String, + record_delimiter: String, + quote_character: String, +} + +#[derive(Clone)] +struct JsonOutputConfig { + record_delimiter: String, +} + +fn parse_select_request(payload: &[u8]) -> Result { + let xml = String::from_utf8_lossy(payload); + let doc = roxmltree::Document::parse(&xml) + .map_err(|_| S3Error::new(S3ErrorCode::MalformedXML, "Unable to parse XML document"))?; + + let root = doc.root_element(); + if root.tag_name().name() != "SelectObjectContentRequest" { + return Err(S3Error::new( + S3ErrorCode::MalformedXML, + "Root element must be SelectObjectContentRequest", + )); + } + + let expression = child_text(&root, "Expression") + .filter(|v| !v.is_empty()) + .ok_or_else(|| S3Error::new(S3ErrorCode::InvalidRequest, "Expression is required"))?; + + let expression_type = child_text(&root, "ExpressionType").unwrap_or_else(|| "SQL".to_string()); + if !expression_type.eq_ignore_ascii_case("SQL") { + return Err(S3Error::new( + S3ErrorCode::InvalidRequest, + "Only SQL expression type is supported", + )); + } + + let input_node = child(&root, "InputSerialization") + .ok_or_else(|| S3Error::new(S3ErrorCode::InvalidRequest, "InputSerialization is required"))?; + let output_node = child(&root, "OutputSerialization") + .ok_or_else(|| S3Error::new(S3ErrorCode::InvalidRequest, "OutputSerialization is required"))?; + + let input_format = parse_input_format(&input_node)?; + let output_format = parse_output_format(&output_node)?; + + Ok(SelectRequest { + expression, + input_format, + output_format, + }) +} + +fn parse_input_format(node: &roxmltree::Node<'_, '_>) -> Result { + if let Some(csv_node) = child(node, "CSV") { + return Ok(InputFormat::Csv(CsvInputConfig { + file_header_info: child_text(&csv_node, "FileHeaderInfo") + .unwrap_or_else(|| "NONE".to_string()) + .to_ascii_uppercase(), + field_delimiter: child_text(&csv_node, "FieldDelimiter").unwrap_or_else(|| ",".to_string()), + quote_character: child_text(&csv_node, "QuoteCharacter").unwrap_or_else(|| "\"".to_string()), + })); + } + + if let Some(json_node) = child(node, "JSON") { + return Ok(InputFormat::Json(JsonInputConfig { + json_type: child_text(&json_node, "Type") + .unwrap_or_else(|| "DOCUMENT".to_string()) + .to_ascii_uppercase(), + })); + } + + if child(node, "Parquet").is_some() { + return Ok(InputFormat::Parquet); + } + + Err(S3Error::new( + S3ErrorCode::InvalidRequest, + "InputSerialization must specify CSV, JSON, or Parquet", + )) +} + +fn parse_output_format(node: &roxmltree::Node<'_, '_>) -> Result { + if let Some(csv_node) = child(node, "CSV") { + return Ok(OutputFormat::Csv(CsvOutputConfig { + field_delimiter: child_text(&csv_node, "FieldDelimiter").unwrap_or_else(|| ",".to_string()), + record_delimiter: child_text(&csv_node, "RecordDelimiter").unwrap_or_else(|| "\n".to_string()), + quote_character: child_text(&csv_node, "QuoteCharacter").unwrap_or_else(|| "\"".to_string()), + })); + } + + if let Some(json_node) = child(node, "JSON") { + return Ok(OutputFormat::Json(JsonOutputConfig { + record_delimiter: child_text(&json_node, "RecordDelimiter").unwrap_or_else(|| "\n".to_string()), + })); + } + + Err(S3Error::new( + S3ErrorCode::InvalidRequest, + "OutputSerialization must specify CSV or JSON", + )) +} + +fn child<'a, 'input>(node: &'a roxmltree::Node<'a, 'input>, name: &str) -> Option> { + node.children() + .find(|n| n.is_element() && n.tag_name().name() == name) +} + +fn child_text(node: &roxmltree::Node<'_, '_>, name: &str) -> Option { + child(node, name) + .and_then(|n| n.text()) + .map(|s| s.to_string()) +} + +fn execute_select_query(path: PathBuf, request: SelectRequest) -> Result>, String> { + let conn = Connection::open_in_memory().map_err(|e| format!("DuckDB connection error: {}", e))?; + + load_input_table(&conn, &path, &request.input_format)?; + + let expression = request + .expression + .replace("s3object", "data") + .replace("S3Object", "data"); + + let mut stmt = conn + .prepare(&expression) + .map_err(|e| format!("SQL execution error: {}", e))?; + let mut rows = stmt + .query([]) + .map_err(|e| format!("SQL execution error: {}", e))?; + let stmt_ref = rows + .as_ref() + .ok_or_else(|| "SQL execution error: statement metadata unavailable".to_string())?; + let col_count = stmt_ref.column_count(); + let mut columns: Vec = Vec::with_capacity(col_count); + for i in 0..col_count { + let name = stmt_ref + .column_name(i) + .map(|s| s.to_string()) + .unwrap_or_else(|_| format!("_{}", i)); + columns.push(name); + } + + match request.output_format { + OutputFormat::Csv(cfg) => collect_csv_chunks(&mut rows, col_count, cfg), + OutputFormat::Json(cfg) => collect_json_chunks(&mut rows, col_count, &columns, cfg), + } +} + +fn load_input_table(conn: &Connection, path: &Path, input: &InputFormat) -> Result<(), String> { + let path_str = path.to_string_lossy().replace('\\', "/"); + match input { + InputFormat::Csv(cfg) => { + let header = cfg.file_header_info == "USE" || cfg.file_header_info == "IGNORE"; + let delimiter = normalize_single_char(&cfg.field_delimiter, ','); + let quote = normalize_single_char(&cfg.quote_character, '"'); + + let sql = format!( + "CREATE TABLE data AS SELECT * FROM read_csv('{}', header={}, delim='{}', quote='{}')", + sql_escape(&path_str), + if header { "true" } else { "false" }, + sql_escape(&delimiter), + sql_escape("e) + ); + conn.execute_batch(&sql) + .map_err(|e| format!("Failed loading CSV data: {}", e))?; + } + InputFormat::Json(cfg) => { + let format = if cfg.json_type == "LINES" { + "newline_delimited" + } else { + "array" + }; + let sql = format!( + "CREATE TABLE data AS SELECT * FROM read_json_auto('{}', format='{}')", + sql_escape(&path_str), + format + ); + conn.execute_batch(&sql) + .map_err(|e| format!("Failed loading JSON data: {}", e))?; + } + InputFormat::Parquet => { + let sql = format!( + "CREATE TABLE data AS SELECT * FROM read_parquet('{}')", + sql_escape(&path_str) + ); + conn.execute_batch(&sql) + .map_err(|e| format!("Failed loading Parquet data: {}", e))?; + } + } + Ok(()) +} + +fn sql_escape(value: &str) -> String { + value.replace('\'', "''") +} + +fn normalize_single_char(value: &str, default_char: char) -> String { + value.chars().next().unwrap_or(default_char).to_string() +} + +fn collect_csv_chunks( + rows: &mut duckdb::Rows<'_>, + col_count: usize, + cfg: CsvOutputConfig, +) -> Result>, String> { + let delimiter = cfg.field_delimiter; + let record_delimiter = cfg.record_delimiter; + let quote = cfg.quote_character; + + let mut chunks: Vec> = Vec::new(); + let mut buffer = String::new(); + + while let Some(row) = rows.next().map_err(|e| format!("SQL execution error: {}", e))? { + let mut fields: Vec = Vec::with_capacity(col_count); + for i in 0..col_count { + let value = row + .get_ref(i) + .map_err(|e| format!("SQL execution error: {}", e))?; + if matches!(value, ValueRef::Null) { + fields.push(String::new()); + continue; + } + + let mut text = value_ref_to_string(value); + if text.contains(&delimiter) || text.contains("e) || text.contains(&record_delimiter) { + text = text.replace("e, &(quote.clone() + "e)); + text = format!("{}{}{}", quote, text, quote); + } + fields.push(text); + } + buffer.push_str(&fields.join(&delimiter)); + buffer.push_str(&record_delimiter); + + while buffer.len() >= CHUNK_SIZE { + let rest = buffer.split_off(CHUNK_SIZE); + chunks.push(buffer.into_bytes()); + buffer = rest; + } + } + + if !buffer.is_empty() { + chunks.push(buffer.into_bytes()); + } + Ok(chunks) +} + +fn collect_json_chunks( + rows: &mut duckdb::Rows<'_>, + col_count: usize, + columns: &[String], + cfg: JsonOutputConfig, +) -> Result>, String> { + let record_delimiter = cfg.record_delimiter; + let mut chunks: Vec> = Vec::new(); + let mut buffer = String::new(); + + while let Some(row) = rows.next().map_err(|e| format!("SQL execution error: {}", e))? { + let mut record: HashMap = HashMap::with_capacity(col_count); + for i in 0..col_count { + let value = row + .get_ref(i) + .map_err(|e| format!("SQL execution error: {}", e))?; + let key = columns + .get(i) + .cloned() + .unwrap_or_else(|| format!("_{}", i)); + record.insert(key, value_ref_to_json(value)); + } + let line = serde_json::to_string(&record) + .map_err(|e| format!("JSON output encoding failed: {}", e))?; + buffer.push_str(&line); + buffer.push_str(&record_delimiter); + + while buffer.len() >= CHUNK_SIZE { + let rest = buffer.split_off(CHUNK_SIZE); + chunks.push(buffer.into_bytes()); + buffer = rest; + } + } + + if !buffer.is_empty() { + chunks.push(buffer.into_bytes()); + } + Ok(chunks) +} + +fn value_ref_to_string(value: ValueRef<'_>) -> String { + match value { + ValueRef::Null => String::new(), + ValueRef::Boolean(v) => v.to_string(), + ValueRef::TinyInt(v) => v.to_string(), + ValueRef::SmallInt(v) => v.to_string(), + ValueRef::Int(v) => v.to_string(), + ValueRef::BigInt(v) => v.to_string(), + ValueRef::UTinyInt(v) => v.to_string(), + ValueRef::USmallInt(v) => v.to_string(), + ValueRef::UInt(v) => v.to_string(), + ValueRef::UBigInt(v) => v.to_string(), + ValueRef::Float(v) => v.to_string(), + ValueRef::Double(v) => v.to_string(), + ValueRef::Decimal(v) => v.to_string(), + ValueRef::Text(v) => String::from_utf8_lossy(v).into_owned(), + ValueRef::Blob(v) => base64::engine::general_purpose::STANDARD.encode(v), + _ => format!("{:?}", value), + } +} + +fn value_ref_to_json(value: ValueRef<'_>) -> serde_json::Value { + match value { + ValueRef::Null => serde_json::Value::Null, + ValueRef::Boolean(v) => serde_json::Value::Bool(v), + ValueRef::TinyInt(v) => serde_json::json!(v), + ValueRef::SmallInt(v) => serde_json::json!(v), + ValueRef::Int(v) => serde_json::json!(v), + ValueRef::BigInt(v) => serde_json::json!(v), + ValueRef::UTinyInt(v) => serde_json::json!(v), + ValueRef::USmallInt(v) => serde_json::json!(v), + ValueRef::UInt(v) => serde_json::json!(v), + ValueRef::UBigInt(v) => serde_json::json!(v), + ValueRef::Float(v) => serde_json::json!(v), + ValueRef::Double(v) => serde_json::json!(v), + ValueRef::Decimal(v) => serde_json::Value::String(v.to_string()), + ValueRef::Text(v) => serde_json::Value::String(String::from_utf8_lossy(v).into_owned()), + ValueRef::Blob(v) => serde_json::Value::String(base64::engine::general_purpose::STANDARD.encode(v)), + _ => serde_json::Value::String(format!("{:?}", value)), + } +} + +fn require_xml_content_type(headers: &HeaderMap) -> Option { + let value = headers + .get("content-type") + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .trim(); + if value.is_empty() { + return None; + } + let lowered = value.to_ascii_lowercase(); + if lowered.starts_with("application/xml") || lowered.starts_with("text/xml") { + return None; + } + Some(s3_error_response(S3Error::new( + S3ErrorCode::InvalidRequest, + "Content-Type must be application/xml or text/xml", + ))) +} + +fn s3_error_response(err: S3Error) -> Response { + let status = StatusCode::from_u16(err.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + let resource = if err.resource.is_empty() { + "/".to_string() + } else { + err.resource.clone() + }; + let body = err + .with_resource(resource) + .with_request_id(uuid::Uuid::new_v4().simple().to_string()) + .to_xml(); + ( + status, + [("content-type", "application/xml")], + body, + ) + .into_response() +} + +fn build_stats_xml(bytes_scanned: usize, bytes_returned: usize) -> String { + format!( + "{}{}{}", + bytes_scanned, + bytes_scanned, + bytes_returned + ) +} + +fn encode_select_event(event_type: &str, payload: &[u8]) -> Vec { + let mut headers = Vec::new(); + headers.extend(encode_select_header(":event-type", event_type)); + if event_type == "Records" { + headers.extend(encode_select_header(":content-type", "application/octet-stream")); + } else if event_type == "Stats" { + headers.extend(encode_select_header(":content-type", "text/xml")); + } + headers.extend(encode_select_header(":message-type", "event")); + + let headers_len = headers.len() as u32; + let total_len = 4 + 4 + 4 + headers.len() + payload.len() + 4; + + let mut message = Vec::with_capacity(total_len); + let mut prelude = Vec::with_capacity(8); + prelude.extend((total_len as u32).to_be_bytes()); + prelude.extend(headers_len.to_be_bytes()); + + let prelude_crc = crc32(&prelude); + message.extend(prelude); + message.extend(prelude_crc.to_be_bytes()); + message.extend(headers); + message.extend(payload); + + let msg_crc = crc32(&message); + message.extend(msg_crc.to_be_bytes()); + message +} + +fn encode_select_header(name: &str, value: &str) -> Vec { + let name_bytes = name.as_bytes(); + let value_bytes = value.as_bytes(); + let mut header = Vec::with_capacity(1 + name_bytes.len() + 1 + 2 + value_bytes.len()); + header.push(name_bytes.len() as u8); + header.extend(name_bytes); + header.push(7); + header.extend((value_bytes.len() as u16).to_be_bytes()); + header.extend(value_bytes); + header +} + +fn crc32(data: &[u8]) -> u32 { + let mut hasher = Hasher::new(); + hasher.update(data); + hasher.finalize() +} diff --git a/myfsio-engine/crates/myfsio-server/src/main.rs b/myfsio-engine/crates/myfsio-server/src/main.rs index 9e2c7c1..12ffdac 100644 --- a/myfsio-engine/crates/myfsio-server/src/main.rs +++ b/myfsio-engine/crates/myfsio-server/src/main.rs @@ -57,13 +57,32 @@ async fn main() { let app = myfsio_server::create_router(state); - let listener = tokio::net::TcpListener::bind(bind_addr).await.unwrap(); + let listener = match tokio::net::TcpListener::bind(bind_addr).await { + Ok(listener) => listener, + Err(err) => { + if err.kind() == std::io::ErrorKind::AddrInUse { + tracing::error!("Port already in use: {}", bind_addr); + } else { + tracing::error!("Failed to bind {}: {}", bind_addr, err); + } + for handle in bg_handles { + handle.abort(); + } + std::process::exit(1); + } + }; tracing::info!("Listening on {}", bind_addr); - axum::serve(listener, app) + if let Err(err) = axum::serve(listener, app) .with_graceful_shutdown(shutdown_signal()) .await - .unwrap(); + { + tracing::error!("Server exited with error: {}", err); + for handle in bg_handles { + handle.abort(); + } + std::process::exit(1); + } for handle in bg_handles { handle.abort(); diff --git a/myfsio-engine/crates/myfsio-server/src/middleware/auth.rs b/myfsio-engine/crates/myfsio-server/src/middleware/auth.rs index 17ab22e..9a1bbef 100644 --- a/myfsio-engine/crates/myfsio-server/src/middleware/auth.rs +++ b/myfsio-engine/crates/myfsio-server/src/middleware/auth.rs @@ -1,5 +1,5 @@ use axum::extract::{Request, State}; -use axum::http::StatusCode; +use axum::http::{Method, StatusCode}; use axum::middleware::Next; use axum::response::{IntoResponse, Response}; @@ -16,17 +16,21 @@ pub async fn auth_layer( next: Next, ) -> Response { let uri = req.uri().clone(); - let path = uri.path(); + let path = uri.path().to_string(); if path == "/" && req.method() == axum::http::Method::GET { match try_auth(&state, &req) { AuthResult::Ok(principal) => { + if let Err(err) = authorize_request(&state, &principal, &req) { + return error_response(err, &path); + } req.extensions_mut().insert(principal); } - AuthResult::Denied(err) => return error_response(err), + AuthResult::Denied(err) => return error_response(err, &path), AuthResult::NoAuth => { return error_response( - S3Error::from_code(S3ErrorCode::AccessDenied), + S3Error::new(S3ErrorCode::AccessDenied, "Missing credentials"), + &path, ); } } @@ -35,12 +39,18 @@ pub async fn auth_layer( match try_auth(&state, &req) { AuthResult::Ok(principal) => { + if let Err(err) = authorize_request(&state, &principal, &req) { + return error_response(err, &path); + } req.extensions_mut().insert(principal); next.run(req).await } - AuthResult::Denied(err) => error_response(err), + AuthResult::Denied(err) => error_response(err, &path), AuthResult::NoAuth => { - error_response(S3Error::from_code(S3ErrorCode::AccessDenied)) + error_response( + S3Error::new(S3ErrorCode::AccessDenied, "Missing credentials"), + &path, + ) } } } @@ -51,6 +61,167 @@ enum AuthResult { NoAuth, } +fn authorize_request(state: &AppState, principal: &Principal, req: &Request) -> Result<(), S3Error> { + let path = req.uri().path(); + if path == "/" { + if state.iam.authorize(principal, None, "list", None) { + return Ok(()); + } + return Err(S3Error::new(S3ErrorCode::AccessDenied, "Access denied")); + } + + let mut segments = path.trim_start_matches('/').split('/').filter(|s| !s.is_empty()); + let bucket = match segments.next() { + Some(b) => b, + None => { + return Err(S3Error::new(S3ErrorCode::AccessDenied, "Access denied")); + } + }; + let remaining: Vec<&str> = segments.collect(); + let query = req.uri().query().unwrap_or(""); + + if remaining.is_empty() { + let action = resolve_bucket_action(req.method(), query); + if state.iam.authorize(principal, Some(bucket), action, None) { + return Ok(()); + } + return Err(S3Error::new(S3ErrorCode::AccessDenied, "Access denied")); + } + + let object_key = remaining.join("/"); + if req.method() == Method::PUT { + if let Some(copy_source) = req + .headers() + .get("x-amz-copy-source") + .and_then(|v| v.to_str().ok()) + { + let source = copy_source.strip_prefix('/').unwrap_or(copy_source); + if let Some((src_bucket, src_key)) = source.split_once('/') { + let source_allowed = + state.iam.authorize(principal, Some(src_bucket), "read", Some(src_key)); + let dest_allowed = + state.iam.authorize(principal, Some(bucket), "write", Some(&object_key)); + if source_allowed && dest_allowed { + return Ok(()); + } + return Err(S3Error::new(S3ErrorCode::AccessDenied, "Access denied")); + } + } + } + + let action = resolve_object_action(req.method(), query); + if state + .iam + .authorize(principal, Some(bucket), action, Some(&object_key)) + { + return Ok(()); + } + + Err(S3Error::new(S3ErrorCode::AccessDenied, "Access denied")) +} + +fn resolve_bucket_action(method: &Method, query: &str) -> &'static str { + if has_query_key(query, "versioning") { + return "versioning"; + } + if has_query_key(query, "tagging") { + return "tagging"; + } + if has_query_key(query, "cors") { + return "cors"; + } + if has_query_key(query, "location") { + return "list"; + } + if has_query_key(query, "encryption") { + return "encryption"; + } + if has_query_key(query, "lifecycle") { + return "lifecycle"; + } + if has_query_key(query, "acl") { + return "share"; + } + if has_query_key(query, "policy") || has_query_key(query, "policyStatus") { + return "policy"; + } + if has_query_key(query, "replication") { + return "replication"; + } + if has_query_key(query, "quota") { + return "quota"; + } + if has_query_key(query, "website") { + return "website"; + } + if has_query_key(query, "object-lock") { + return "object_lock"; + } + if has_query_key(query, "notification") { + return "notification"; + } + if has_query_key(query, "logging") { + return "logging"; + } + if has_query_key(query, "versions") || has_query_key(query, "uploads") { + return "list"; + } + if has_query_key(query, "delete") { + return "delete"; + } + + match *method { + Method::GET => "list", + Method::HEAD => "read", + Method::PUT => "create_bucket", + Method::DELETE => "delete_bucket", + Method::POST => "write", + _ => "list", + } +} + +fn resolve_object_action(method: &Method, query: &str) -> &'static str { + if has_query_key(query, "tagging") { + return if *method == Method::GET { "read" } else { "write" }; + } + if has_query_key(query, "acl") { + return if *method == Method::GET { "read" } else { "write" }; + } + if has_query_key(query, "retention") || has_query_key(query, "legal-hold") { + return "object_lock"; + } + if has_query_key(query, "attributes") { + return "read"; + } + if has_query_key(query, "uploads") || has_query_key(query, "uploadId") { + return match *method { + Method::GET => "read", + _ => "write", + }; + } + if has_query_key(query, "select") { + return "read"; + } + + match *method { + Method::GET | Method::HEAD => "read", + Method::PUT => "write", + Method::DELETE => "delete", + Method::POST => "write", + _ => "read", + } +} + +fn has_query_key(query: &str, key: &str) -> bool { + if query.is_empty() { + return false; + } + query + .split('&') + .filter(|part| !part.is_empty()) + .any(|part| part == key || part.starts_with(&format!("{}=", key))) +} + fn try_auth(state: &AppState, req: &Request) -> AuthResult { if let Some(auth_header) = req.headers().get("authorization") { if let Ok(auth_str) = auth_header.to_str() { @@ -382,9 +553,13 @@ fn urlencoding_decode(s: &str) -> String { .into_owned() } -fn error_response(err: S3Error) -> Response { +fn error_response(err: S3Error, resource: &str) -> Response { let status = StatusCode::from_u16(err.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); - let body = err.to_xml(); + let request_id = uuid::Uuid::new_v4().simple().to_string(); + let body = err + .with_resource(resource.to_string()) + .with_request_id(request_id) + .to_xml(); (status, [("content-type", "application/xml")], body).into_response() } diff --git a/myfsio-engine/crates/myfsio-server/tests/integration.rs b/myfsio-engine/crates/myfsio-server/tests/integration.rs index 28d21cc..e9a2fbf 100644 --- a/myfsio-engine/crates/myfsio-server/tests/integration.rs +++ b/myfsio-engine/crates/myfsio-server/tests/integration.rs @@ -1,34 +1,17 @@ use axum::body::Body; use axum::http::{Method, Request, StatusCode}; use http_body_util::BodyExt; +use myfsio_storage::traits::StorageEngine; use tower::ServiceExt; const TEST_ACCESS_KEY: &str = "AKIAIOSFODNN7EXAMPLE"; const TEST_SECRET_KEY: &str = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; -fn test_app() -> (axum::Router, tempfile::TempDir) { +fn test_app_with_iam(iam_json: serde_json::Value) -> (axum::Router, tempfile::TempDir) { let tmp = tempfile::TempDir::new().unwrap(); let iam_path = tmp.path().join(".myfsio.sys").join("config"); std::fs::create_dir_all(&iam_path).unwrap(); - let iam_json = serde_json::json!({ - "version": 2, - "users": [{ - "user_id": "u-test1234", - "display_name": "admin", - "enabled": true, - "access_keys": [{ - "access_key": TEST_ACCESS_KEY, - "secret_key": TEST_SECRET_KEY, - "status": "active" - }], - "policies": [{ - "bucket": "*", - "actions": ["*"], - "prefix": "*" - }] - }] - }); std::fs::write(iam_path.join("iam.json"), iam_json.to_string()).unwrap(); let config = myfsio_server::config::ServerConfig { @@ -52,6 +35,27 @@ fn test_app() -> (axum::Router, tempfile::TempDir) { (app, tmp) } +fn test_app() -> (axum::Router, tempfile::TempDir) { + test_app_with_iam(serde_json::json!({ + "version": 2, + "users": [{ + "user_id": "u-test1234", + "display_name": "admin", + "enabled": true, + "access_keys": [{ + "access_key": TEST_ACCESS_KEY, + "secret_key": TEST_SECRET_KEY, + "status": "active" + }], + "policies": [{ + "bucket": "*", + "actions": ["*"], + "prefix": "*" + }] + }] + })) +} + fn signed_request(method: Method, uri: &str, body: Body) -> Request { Request::builder() .method(method) @@ -62,6 +66,75 @@ fn signed_request(method: Method, uri: &str, body: Body) -> Request { .unwrap() } +fn parse_select_events(body: &[u8]) -> Vec<(String, Vec)> { + let mut out = Vec::new(); + let mut idx: usize = 0; + + while idx + 16 <= body.len() { + let total_len = u32::from_be_bytes([ + body[idx], + body[idx + 1], + body[idx + 2], + body[idx + 3], + ]) as usize; + let headers_len = u32::from_be_bytes([ + body[idx + 4], + body[idx + 5], + body[idx + 6], + body[idx + 7], + ]) as usize; + if total_len < 16 || idx + total_len > body.len() { + break; + } + + let headers_start = idx + 12; + let headers_end = headers_start + headers_len; + if headers_end > idx + total_len - 4 { + break; + } + + let mut event_type: Option = None; + let mut hidx = headers_start; + while hidx < headers_end { + let name_len = body[hidx] as usize; + hidx += 1; + if hidx + name_len + 3 > headers_end { + break; + } + let name = String::from_utf8_lossy(&body[hidx..hidx + name_len]).to_string(); + hidx += name_len; + + let value_type = body[hidx]; + hidx += 1; + if value_type != 7 || hidx + 2 > headers_end { + break; + } + + let value_len = u16::from_be_bytes([body[hidx], body[hidx + 1]]) as usize; + hidx += 2; + if hidx + value_len > headers_end { + break; + } + + let value = String::from_utf8_lossy(&body[hidx..hidx + value_len]).to_string(); + hidx += value_len; + + if name == ":event-type" { + event_type = Some(value); + } + } + + let payload_start = headers_end; + let payload_end = idx + total_len - 4; + let payload = body[payload_start..payload_end].to_vec(); + + out.push((event_type.unwrap_or_default(), payload)); + idx += total_len; + } + + out +} + #[tokio::test] async fn test_unauthenticated_request_rejected() { let (app, _tmp) = test_app(); @@ -70,6 +143,34 @@ async fn test_unauthenticated_request_rejected() { .await .unwrap(); assert_eq!(resp.status(), StatusCode::FORBIDDEN); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("AccessDenied")); + assert!(body.contains("Missing credentials")); + assert!(body.contains("/")); + assert!(body.contains("")); + assert!(!body.contains("")); +} + +#[tokio::test] +async fn test_unauthenticated_request_includes_requested_resource_path() { + let (app, _tmp) = test_app(); + let resp = app + .oneshot(Request::builder().uri("/ui/").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::FORBIDDEN); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("AccessDenied")); + assert!(body.contains("Missing credentials")); + assert!(body.contains("/ui/")); + assert!(body.contains("")); + assert!(!body.contains("")); } #[tokio::test] @@ -200,6 +301,54 @@ async fn test_put_and_get_object() { assert_eq!(&body[..], b"Hello, World!"); } +#[tokio::test] +async fn test_content_type_falls_back_to_extension() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/img-bucket", Body::empty())) + .await + .unwrap(); + + let resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/img-bucket/yum.jpg") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from(vec![0_u8, 1, 2, 3, 4])) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let resp = app + .clone() + .oneshot(signed_request( + Method::GET, + "/img-bucket/yum.jpg", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(resp.headers().get("content-type").unwrap(), "image/jpeg"); + + let resp = app + .oneshot(signed_request( + Method::HEAD, + "/img-bucket/yum.jpg", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(resp.headers().get("content-type").unwrap(), "image/jpeg"); +} + #[tokio::test] async fn test_head_object() { let (app, _tmp) = test_app(); @@ -1189,6 +1338,642 @@ async fn test_object_legal_hold() { assert!(body.contains("OFF")); } +#[tokio::test] +async fn test_list_objects_v1_marker_flow() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/v1-bucket", Body::empty())) + .await + .unwrap(); + + for name in ["a.txt", "b.txt", "c.txt"] { + app.clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri(format!("/v1-bucket/{}", name)) + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from("data")) + .unwrap(), + ) + .await + .unwrap(); + } + + let resp = app + .clone() + .oneshot(signed_request( + Method::GET, + "/v1-bucket?max-keys=2", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8(resp.into_body().collect().await.unwrap().to_bytes().to_vec()).unwrap(); + assert!(body.contains("")); + assert!(body.contains("true") || body.contains("false")); +} + +#[tokio::test] +async fn test_bucket_quota_roundtrip() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/quota-bucket", Body::empty())) + .await + .unwrap(); + + let resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/quota-bucket?quota") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("content-type", "application/json") + .body(Body::from(r#"{"max_size_bytes": 1024, "max_objects": 10}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let resp = app + .clone() + .oneshot(signed_request( + Method::GET, + "/quota-bucket?quota", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body: serde_json::Value = serde_json::from_slice(&resp.into_body().collect().await.unwrap().to_bytes()).unwrap(); + assert_eq!(body["quota"]["max_size_bytes"], 1024); + assert_eq!(body["quota"]["max_objects"], 10); + + let resp = app + .oneshot(signed_request( + Method::DELETE, + "/quota-bucket?quota", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); +} + +#[tokio::test] +async fn test_bucket_policy_and_status_roundtrip() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/policy-bucket", Body::empty())) + .await + .unwrap(); + + let policy = r#"{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": "*", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::policy-bucket/*" + }] + }"#; + + let resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/policy-bucket?policy") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("content-type", "application/json") + .body(Body::from(policy)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let resp = app + .clone() + .oneshot(signed_request( + Method::GET, + "/policy-bucket?policy", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body: serde_json::Value = serde_json::from_slice(&resp.into_body().collect().await.unwrap().to_bytes()).unwrap(); + assert!(body.get("Statement").is_some()); + + let resp = app + .clone() + .oneshot(signed_request( + Method::GET, + "/policy-bucket?policyStatus", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8(resp.into_body().collect().await.unwrap().to_bytes().to_vec()).unwrap(); + assert!(body.contains("TRUE")); + + let resp = app + .oneshot(signed_request( + Method::DELETE, + "/policy-bucket?policy", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); +} + +#[tokio::test] +async fn test_bucket_replication_roundtrip() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/repl-bucket", Body::empty())) + .await + .unwrap(); + + let repl_xml = "arn:aws:iam::123456789012:role/s3-replrule-1"; + + let resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/repl-bucket?replication") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("content-type", "application/xml") + .body(Body::from(repl_xml)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let resp = app + .clone() + .oneshot(signed_request( + Method::GET, + "/repl-bucket?replication", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8(resp.into_body().collect().await.unwrap().to_bytes().to_vec()).unwrap(); + assert!(body.contains("ReplicationConfiguration")); + + let resp = app + .oneshot(signed_request( + Method::DELETE, + "/repl-bucket?replication", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); +} + +#[tokio::test] +async fn test_list_parts_via_get_upload_id() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/parts-bucket", Body::empty())) + .await + .unwrap(); + + let resp = app + .clone() + .oneshot(signed_request( + Method::POST, + "/parts-bucket/large.bin?uploads", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8(resp.into_body().collect().await.unwrap().to_bytes().to_vec()).unwrap(); + let upload_id = body + .split("") + .nth(1) + .unwrap() + .split("") + .next() + .unwrap(); + + let resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri(format!("/parts-bucket/large.bin?uploadId={}&partNumber=1", upload_id)) + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from(vec![1_u8, 2, 3, 4])) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let resp = app + .oneshot(signed_request( + Method::GET, + &format!("/parts-bucket/large.bin?uploadId={}", upload_id), + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8(resp.into_body().collect().await.unwrap().to_bytes().to_vec()).unwrap(); + assert!(body.contains("ListPartsResult")); + assert!(body.contains("1")); +} + +#[tokio::test] +async fn test_conditional_get_and_head() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/cond-bucket", Body::empty())) + .await + .unwrap(); + + let put_resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/cond-bucket/item.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from("abc")) + .unwrap(), + ) + .await + .unwrap(); + let etag = put_resp.headers().get("etag").unwrap().to_str().unwrap().to_string(); + + let resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::GET) + .uri("/cond-bucket/item.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("if-none-match", etag.as_str()) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_MODIFIED); + + let resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::GET) + .uri("/cond-bucket/item.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("if-match", "\"does-not-match\"") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::PRECONDITION_FAILED); + + let resp = app + .oneshot( + Request::builder() + .method(Method::HEAD) + .uri("/cond-bucket/item.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("if-none-match", etag.as_str()) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_MODIFIED); +} + +#[tokio::test] +async fn test_copy_source_preconditions() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/src-pre", Body::empty())) + .await + .unwrap(); + app.clone() + .oneshot(signed_request(Method::PUT, "/dst-pre", Body::empty())) + .await + .unwrap(); + + let put_resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/src-pre/original.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from("copy source")) + .unwrap(), + ) + .await + .unwrap(); + let etag = put_resp.headers().get("etag").unwrap().to_str().unwrap().to_string(); + + let resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/dst-pre/copied.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("x-amz-copy-source", "/src-pre/original.txt") + .header("x-amz-copy-source-if-match", "\"bad-etag\"") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::PRECONDITION_FAILED); + + let resp = app + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/dst-pre/copied.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("x-amz-copy-source", "/src-pre/original.txt") + .header("x-amz-copy-source-if-match", etag.as_str()) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); +} + +#[tokio::test] +async fn test_select_object_content_csv_to_json_events() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/sel-bucket", Body::empty())) + .await + .unwrap(); + + app.clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/sel-bucket/people.csv") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("content-type", "text/csv") + .body(Body::from("name,age\nalice,30\nbob,40\n")) + .unwrap(), + ) + .await + .unwrap(); + + let select_xml = r#" + + SELECT name, age FROM S3Object WHERE CAST(age AS INTEGER) >= 35 + SQL + + + USE + + + + + \n + + + +"#; + + let resp = app + .oneshot( + Request::builder() + .method(Method::POST) + .uri("/sel-bucket/people.csv?select&select-type=2") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("content-type", "application/xml") + .body(Body::from(select_xml)) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(resp.headers().get("content-type").unwrap(), "application/octet-stream"); + assert_eq!(resp.headers().get("x-amz-request-charged").unwrap(), "requester"); + + let body = resp.into_body().collect().await.unwrap().to_bytes(); + let events = parse_select_events(&body); + assert!(events.iter().any(|(name, _)| name == "Records")); + assert!(events.iter().any(|(name, _)| name == "Stats")); + assert!(events.iter().any(|(name, _)| name == "End")); + + let mut records = String::new(); + for (name, payload) in events { + if name == "Records" { + records.push_str(&String::from_utf8_lossy(&payload)); + } + } + assert!(records.contains("bob")); + assert!(!records.contains("alice")); +} + +#[tokio::test] +async fn test_select_object_content_requires_expression() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/sel-missing-exp", Body::empty())) + .await + .unwrap(); + app.clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/sel-missing-exp/file.csv") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from("a,b\n1,2\n")) + .unwrap(), + ) + .await + .unwrap(); + + let select_xml = r#" + + SQL + USE + + +"#; + + let resp = app + .oneshot( + Request::builder() + .method(Method::POST) + .uri("/sel-missing-exp/file.csv?select") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("content-type", "application/xml") + .body(Body::from(select_xml)) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + let body = String::from_utf8(resp.into_body().collect().await.unwrap().to_bytes().to_vec()).unwrap(); + assert!(body.contains("InvalidRequest")); + assert!(body.contains("Expression is required")); +} + +#[tokio::test] +async fn test_select_object_content_rejects_non_xml_content_type() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/sel-ct", Body::empty())) + .await + .unwrap(); + app.clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/sel-ct/file.csv") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from("a,b\n1,2\n")) + .unwrap(), + ) + .await + .unwrap(); + + let select_xml = r#" + + SELECT * FROM S3Object + SQL + USE + + +"#; + + let resp = app + .oneshot( + Request::builder() + .method(Method::POST) + .uri("/sel-ct/file.csv?select") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("content-type", "application/json") + .body(Body::from(select_xml)) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + let body = String::from_utf8(resp.into_body().collect().await.unwrap().to_bytes().to_vec()).unwrap(); + assert!(body.contains("InvalidRequest")); + assert!(body.contains("Content-Type must be application/xml or text/xml")); +} + +#[tokio::test] +async fn test_non_admin_authorization_enforced() { + let iam_json = serde_json::json!({ + "version": 2, + "users": [{ + "user_id": "u-limited", + "display_name": "limited", + "enabled": true, + "access_keys": [{ + "access_key": TEST_ACCESS_KEY, + "secret_key": TEST_SECRET_KEY, + "status": "active" + }], + "policies": [{ + "bucket": "authz-bucket", + "actions": ["list", "read"], + "prefix": "*" + }] + }] + }); + + let tmp = tempfile::TempDir::new().unwrap(); + let iam_path = tmp.path().join(".myfsio.sys").join("config"); + std::fs::create_dir_all(&iam_path).unwrap(); + std::fs::write(iam_path.join("iam.json"), iam_json.to_string()).unwrap(); + + let config = myfsio_server::config::ServerConfig { + bind_addr: "127.0.0.1:0".parse().unwrap(), + storage_root: tmp.path().to_path_buf(), + region: "us-east-1".to_string(), + iam_config_path: iam_path.join("iam.json"), + sigv4_timestamp_tolerance_secs: 900, + presigned_url_min_expiry: 1, + presigned_url_max_expiry: 604800, + secret_key: None, + encryption_enabled: false, + kms_enabled: false, + gc_enabled: false, + integrity_enabled: false, + metrics_enabled: false, + lifecycle_enabled: false, + }; + let state = myfsio_server::state::AppState::new(config); + state.storage.create_bucket("authz-bucket").await.unwrap(); + let app = myfsio_server::create_router(state); + + let resp = app + .clone() + .oneshot(signed_request(Method::PUT, "/denied-bucket", Body::empty())) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::FORBIDDEN); + + let resp = app + .oneshot(signed_request(Method::GET, "/authz-bucket", Body::empty())) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); +} + async fn test_app_encrypted() -> (axum::Router, tempfile::TempDir) { let tmp = tempfile::TempDir::new().unwrap(); let iam_path = tmp.path().join(".myfsio.sys").join("config"); diff --git a/myfsio-engine/crates/myfsio-xml/src/response.rs b/myfsio-engine/crates/myfsio-xml/src/response.rs index 18b086d..791e3f0 100644 --- a/myfsio-engine/crates/myfsio-xml/src/response.rs +++ b/myfsio-engine/crates/myfsio-xml/src/response.rs @@ -90,6 +90,76 @@ pub fn list_objects_v2_xml( String::from_utf8(writer.into_inner().into_inner()).unwrap() } +pub fn list_objects_v1_xml( + bucket_name: &str, + prefix: &str, + marker: &str, + delimiter: &str, + max_keys: usize, + objects: &[ObjectMeta], + common_prefixes: &[String], + is_truncated: bool, + next_marker: Option<&str>, +) -> String { + let mut writer = Writer::new(Cursor::new(Vec::new())); + + writer + .write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))) + .unwrap(); + + let start = BytesStart::new("ListBucketResult") + .with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]); + writer.write_event(Event::Start(start)).unwrap(); + + write_text_element(&mut writer, "Name", bucket_name); + write_text_element(&mut writer, "Prefix", prefix); + write_text_element(&mut writer, "Marker", marker); + write_text_element(&mut writer, "MaxKeys", &max_keys.to_string()); + write_text_element(&mut writer, "IsTruncated", &is_truncated.to_string()); + + if !delimiter.is_empty() { + write_text_element(&mut writer, "Delimiter", delimiter); + } + if !delimiter.is_empty() && is_truncated { + if let Some(nm) = next_marker { + if !nm.is_empty() { + write_text_element(&mut writer, "NextMarker", nm); + } + } + } + + for obj in objects { + writer + .write_event(Event::Start(BytesStart::new("Contents"))) + .unwrap(); + write_text_element(&mut writer, "Key", &obj.key); + write_text_element(&mut writer, "LastModified", &obj.last_modified.to_rfc3339()); + if let Some(ref etag) = obj.etag { + write_text_element(&mut writer, "ETag", &format!("\"{}\"", etag)); + } + write_text_element(&mut writer, "Size", &obj.size.to_string()); + writer + .write_event(Event::End(BytesEnd::new("Contents"))) + .unwrap(); + } + + for cp in common_prefixes { + writer + .write_event(Event::Start(BytesStart::new("CommonPrefixes"))) + .unwrap(); + write_text_element(&mut writer, "Prefix", cp); + writer + .write_event(Event::End(BytesEnd::new("CommonPrefixes"))) + .unwrap(); + } + + writer + .write_event(Event::End(BytesEnd::new("ListBucketResult"))) + .unwrap(); + + String::from_utf8(writer.into_inner().into_inner()).unwrap() +} + fn write_text_element(writer: &mut Writer>>, tag: &str, text: &str) { writer.write_event(Event::Start(BytesStart::new(tag))).unwrap(); writer.write_event(Event::Text(BytesText::new(text))).unwrap(); @@ -266,4 +336,23 @@ mod tests { assert!(xml.contains("1024")); assert!(xml.contains("false")); } + + #[test] + fn test_list_objects_v1_xml() { + let objects = vec![ObjectMeta::new("file.txt".to_string(), 1024, Utc::now())]; + let xml = list_objects_v1_xml( + "my-bucket", + "", + "", + "/", + 1000, + &objects, + &[], + false, + None, + ); + assert!(xml.contains("file.txt")); + assert!(xml.contains("1024")); + assert!(xml.contains("")); + } }