diff --git a/.gitignore b/.gitignore index 210bd29..4cc3147 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,9 @@ dist/ myfsio_core/target/ myfsio_core/Cargo.lock +# Rust engine build artifacts +myfsio-engine/target/ + # Local runtime artifacts logs/ *.log diff --git a/app/version.py b/app/version.py index 0abc36a..d2d65c4 100644 --- a/app/version.py +++ b/app/version.py @@ -1,6 +1,6 @@ from __future__ import annotations -APP_VERSION = "0.4.2" +APP_VERSION = "0.4.3" def get_version() -> str: diff --git a/myfsio-engine/Cargo.lock b/myfsio-engine/Cargo.lock new file mode 100644 index 0000000..596acf6 --- /dev/null +++ b/myfsio-engine/Cargo.lock @@ -0,0 +1,1807 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.2.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core", + "typenum", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.184" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f8cc7106155f10bdf99a6f379688f543ad6596a415375b36a59a054ceda1198" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "myfsio-auth" +version = "0.1.0" +dependencies = [ + "aes", + "base64", + "cbc", + "chrono", + "hex", + "hmac", + "lru", + "myfsio-common", + "parking_lot", + "pbkdf2", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "tempfile", + "thiserror", + "tracing", +] + +[[package]] +name = "myfsio-common" +version = "0.1.0" +dependencies = [ + "chrono", + "serde", + "serde_json", + "thiserror", + "uuid", +] + +[[package]] +name = "myfsio-crypto" +version = "0.1.0" +dependencies = [ + "aes-gcm", + "base64", + "chrono", + "hex", + "hkdf", + "md-5", + "myfsio-common", + "rand", + "serde", + "serde_json", + "sha2", + "tempfile", + "thiserror", + "tokio", + "uuid", +] + +[[package]] +name = "myfsio-server" +version = "0.1.0" +dependencies = [ + "axum", + "base64", + "bytes", + "chrono", + "futures", + "http-body-util", + "hyper", + "myfsio-auth", + "myfsio-common", + "myfsio-crypto", + "myfsio-storage", + "myfsio-xml", + "percent-encoding", + "quick-xml", + "serde", + "serde_json", + "tempfile", + "tokio", + "tokio-util", + "tower", + "tower-http", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "myfsio-storage" +version = "0.1.0" +dependencies = [ + "chrono", + "dashmap", + "hex", + "md-5", + "myfsio-common", + "myfsio-crypto", + "parking_lot", + "regex", + "serde", + "serde_json", + "sha2", + "tempfile", + "thiserror", + "tokio", + "tracing", + "unicode-normalization", + "uuid", +] + +[[package]] +name = "myfsio-xml" +version = "0.1.0" +dependencies = [ + "chrono", + "myfsio-common", + "quick-xml", + "serde", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags", + "bytes", + "http", + "http-body", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + +[[package]] +name = "uuid" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/myfsio-engine/Cargo.toml b/myfsio-engine/Cargo.toml new file mode 100644 index 0000000..45a02a8 --- /dev/null +++ b/myfsio-engine/Cargo.toml @@ -0,0 +1,43 @@ +[workspace] +resolver = "2" +members = [ + "crates/myfsio-common", + "crates/myfsio-auth", + "crates/myfsio-crypto", + "crates/myfsio-storage", + "crates/myfsio-xml", + "crates/myfsio-server", +] + +[workspace.dependencies] +tokio = { version = "1", features = ["full"] } +axum = { version = "0.8" } +tower = { version = "0.5" } +tower-http = { version = "0.6", features = ["cors", "trace"] } +hyper = { version = "1" } +bytes = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +quick-xml = { version = "0.37", features = ["serialize"] } +hmac = "0.12" +sha2 = "0.10" +md-5 = "0.10" +hex = "0.4" +aes = "0.8" +aes-gcm = "0.10" +cbc = { version = "0.1", features = ["alloc"] } +hkdf = "0.12" +uuid = { version = "1", features = ["v4"] } +parking_lot = "0.12" +lru = "0.14" +percent-encoding = "2" +regex = "1" +unicode-normalization = "0.1" +tracing = "0.1" +tracing-subscriber = "0.3" +thiserror = "2" +chrono = { version = "0.4", features = ["serde"] } +base64 = "0.22" +tokio-util = { version = "0.7", features = ["io"] } +futures = "0.3" +dashmap = "6" diff --git a/myfsio-engine/crates/myfsio-auth/Cargo.toml b/myfsio-engine/crates/myfsio-auth/Cargo.toml new file mode 100644 index 0000000..0676e6e --- /dev/null +++ b/myfsio-engine/crates/myfsio-auth/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "myfsio-auth" +version = "0.1.0" +edition = "2021" + +[dependencies] +myfsio-common = { path = "../myfsio-common" } +hmac = { workspace = true } +sha2 = { workspace = true } +hex = { workspace = true } +aes = { workspace = true } +cbc = { workspace = true } +base64 = { workspace = true } +pbkdf2 = "0.12" +lru = { workspace = true } +parking_lot = { workspace = true } +percent-encoding = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +chrono = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +tempfile = "3" diff --git a/myfsio-engine/crates/myfsio-auth/src/fernet.rs b/myfsio-engine/crates/myfsio-auth/src/fernet.rs new file mode 100644 index 0000000..ba7fb64 --- /dev/null +++ b/myfsio-engine/crates/myfsio-auth/src/fernet.rs @@ -0,0 +1,80 @@ +use aes::cipher::{block_padding::Pkcs7, BlockDecryptMut, KeyIvInit}; +use base64::{engine::general_purpose::URL_SAFE, Engine}; +use hmac::{Hmac, Mac}; +use sha2::Sha256; + +type Aes128CbcDec = cbc::Decryptor; +type HmacSha256 = Hmac; + +pub fn derive_fernet_key(secret: &str) -> String { + let mut derived = [0u8; 32]; + pbkdf2::pbkdf2_hmac::( + secret.as_bytes(), + b"myfsio-iam-encryption", + 100_000, + &mut derived, + ); + URL_SAFE.encode(derived) +} + +pub fn decrypt(key_b64: &str, token: &str) -> Result, &'static str> { + let key_bytes = URL_SAFE + .decode(key_b64) + .map_err(|_| "invalid fernet key base64")?; + if key_bytes.len() != 32 { + return Err("fernet key must be 32 bytes"); + } + + let signing_key = &key_bytes[..16]; + let encryption_key = &key_bytes[16..]; + + let token_bytes = URL_SAFE + .decode(token) + .map_err(|_| "invalid fernet token base64")?; + + if token_bytes.len() < 57 { + return Err("fernet token too short"); + } + + if token_bytes[0] != 0x80 { + return Err("invalid fernet version"); + } + + let hmac_offset = token_bytes.len() - 32; + let payload = &token_bytes[..hmac_offset]; + let expected_hmac = &token_bytes[hmac_offset..]; + + let mut mac = + HmacSha256::new_from_slice(signing_key).map_err(|_| "hmac key error")?; + mac.update(payload); + mac.verify_slice(expected_hmac) + .map_err(|_| "HMAC verification failed")?; + + let iv = &token_bytes[9..25]; + let ciphertext = &token_bytes[25..hmac_offset]; + + let plaintext = Aes128CbcDec::new(encryption_key.into(), iv.into()) + .decrypt_padded_vec_mut::(ciphertext) + .map_err(|_| "AES-CBC decryption failed")?; + + Ok(plaintext) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_derive_fernet_key_format() { + let key = derive_fernet_key("test-secret"); + let decoded = URL_SAFE.decode(&key).unwrap(); + assert_eq!(decoded.len(), 32); + } + + #[test] + fn test_roundtrip_with_python_compat() { + let key = derive_fernet_key("dev-secret-key"); + let decoded = URL_SAFE.decode(&key).unwrap(); + assert_eq!(decoded.len(), 32); + } +} diff --git a/myfsio-engine/crates/myfsio-auth/src/iam.rs b/myfsio-engine/crates/myfsio-auth/src/iam.rs new file mode 100644 index 0000000..0a9071d --- /dev/null +++ b/myfsio-engine/crates/myfsio-auth/src/iam.rs @@ -0,0 +1,499 @@ +use chrono::{DateTime, Utc}; +use myfsio_common::types::Principal; +use parking_lot::RwLock; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Instant, SystemTime}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IamConfig { + #[serde(default = "default_version")] + pub version: u32, + #[serde(default)] + pub users: Vec, +} + +fn default_version() -> u32 { + 2 +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IamUser { + pub user_id: String, + pub display_name: String, + #[serde(default = "default_enabled")] + pub enabled: bool, + #[serde(default)] + pub expires_at: Option, + #[serde(default)] + pub access_keys: Vec, + #[serde(default)] + pub policies: Vec, +} + +fn default_enabled() -> bool { + true +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AccessKey { + pub access_key: String, + pub secret_key: String, + #[serde(default = "default_status")] + pub status: String, + #[serde(default)] + pub created_at: Option, +} + +fn default_status() -> String { + "active".to_string() +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IamPolicy { + pub bucket: String, + pub actions: Vec, + #[serde(default = "default_prefix")] + pub prefix: String, +} + +fn default_prefix() -> String { + "*".to_string() +} + +struct IamState { + key_secrets: HashMap, + key_index: HashMap, + key_status: HashMap, + user_records: HashMap, + file_mtime: Option, + last_check: Instant, +} + +pub struct IamService { + config_path: PathBuf, + state: Arc>, + check_interval: std::time::Duration, + fernet_key: Option, +} + +impl IamService { + pub fn new(config_path: PathBuf) -> Self { + Self::new_with_secret(config_path, None) + } + + pub fn new_with_secret(config_path: PathBuf, secret_key: Option) -> Self { + let fernet_key = secret_key.map(|s| crate::fernet::derive_fernet_key(&s)); + let service = Self { + config_path, + state: Arc::new(RwLock::new(IamState { + key_secrets: HashMap::new(), + key_index: HashMap::new(), + key_status: HashMap::new(), + user_records: HashMap::new(), + file_mtime: None, + last_check: Instant::now(), + })), + check_interval: std::time::Duration::from_secs(2), + fernet_key, + }; + service.reload(); + service + } + + fn reload_if_needed(&self) { + { + let state = self.state.read(); + if state.last_check.elapsed() < self.check_interval { + return; + } + } + + let current_mtime = std::fs::metadata(&self.config_path) + .and_then(|m| m.modified()) + .ok(); + + let needs_reload = { + let state = self.state.read(); + match (&state.file_mtime, ¤t_mtime) { + (None, Some(_)) => true, + (Some(old), Some(new)) => old != new, + (Some(_), None) => true, + (None, None) => state.key_secrets.is_empty(), + } + }; + + if needs_reload { + self.reload(); + } + + self.state.write().last_check = Instant::now(); + } + + fn reload(&self) { + let content = match std::fs::read_to_string(&self.config_path) { + Ok(c) => c, + Err(e) => { + tracing::warn!("Failed to read IAM config {}: {}", self.config_path.display(), e); + return; + } + }; + + let raw = if content.starts_with("MYFSIO_IAM_ENC:") { + let encrypted_token = &content["MYFSIO_IAM_ENC:".len()..]; + match &self.fernet_key { + Some(key) => match crate::fernet::decrypt(key, encrypted_token.trim()) { + Ok(plaintext) => match String::from_utf8(plaintext) { + Ok(s) => s, + Err(e) => { + tracing::error!("Decrypted IAM config is not valid UTF-8: {}", e); + return; + } + }, + Err(e) => { + tracing::error!("Failed to decrypt IAM config: {}. SECRET_KEY may have changed.", e); + return; + } + }, + None => { + tracing::error!("IAM config is encrypted but no SECRET_KEY configured"); + return; + } + } + } else { + content + }; + + let config: IamConfig = match serde_json::from_str(&raw) { + Ok(c) => c, + Err(e) => { + tracing::error!("Failed to parse IAM config: {}", e); + return; + } + }; + + let mut key_secrets = HashMap::new(); + let mut key_index = HashMap::new(); + let mut key_status = HashMap::new(); + let mut user_records = HashMap::new(); + + for user in &config.users { + user_records.insert(user.user_id.clone(), user.clone()); + for ak in &user.access_keys { + key_secrets.insert(ak.access_key.clone(), ak.secret_key.clone()); + key_index.insert(ak.access_key.clone(), user.user_id.clone()); + key_status.insert(ak.access_key.clone(), ak.status.clone()); + } + } + + let file_mtime = std::fs::metadata(&self.config_path) + .and_then(|m| m.modified()) + .ok(); + + let mut state = self.state.write(); + state.key_secrets = key_secrets; + state.key_index = key_index; + state.key_status = key_status; + state.user_records = user_records; + state.file_mtime = file_mtime; + state.last_check = Instant::now(); + + tracing::info!("IAM config reloaded: {} users, {} keys", + config.users.len(), + state.key_secrets.len()); + } + + pub fn get_secret_key(&self, access_key: &str) -> Option { + self.reload_if_needed(); + let state = self.state.read(); + + let status = state.key_status.get(access_key)?; + if status != "active" { + return None; + } + + let user_id = state.key_index.get(access_key)?; + let user = state.user_records.get(user_id)?; + if !user.enabled { + return None; + } + + if let Some(ref expires_at) = user.expires_at { + if let Ok(exp) = expires_at.parse::>() { + if Utc::now() > exp { + return None; + } + } + } + + state.key_secrets.get(access_key).cloned() + } + + pub fn get_principal(&self, access_key: &str) -> Option { + self.reload_if_needed(); + let state = self.state.read(); + + let status = state.key_status.get(access_key)?; + if status != "active" { + return None; + } + + let user_id = state.key_index.get(access_key)?; + let user = state.user_records.get(user_id)?; + if !user.enabled { + return None; + } + + if let Some(ref expires_at) = user.expires_at { + if let Ok(exp) = expires_at.parse::>() { + if Utc::now() > exp { + return None; + } + } + } + + let is_admin = user.policies.iter().any(|p| { + p.bucket == "*" && p.actions.iter().any(|a| a == "*") + }); + + Some(Principal::new( + access_key.to_string(), + user.user_id.clone(), + user.display_name.clone(), + is_admin, + )) + } + + pub fn authenticate(&self, access_key: &str, secret_key: &str) -> Option { + let stored_secret = self.get_secret_key(access_key)?; + if !crate::sigv4::constant_time_compare(&stored_secret, secret_key) { + return None; + } + self.get_principal(access_key) + } + + pub async fn list_users(&self) -> Vec { + self.reload_if_needed(); + let state = self.state.read(); + state + .user_records + .values() + .map(|u| { + serde_json::json!({ + "user_id": u.user_id, + "display_name": u.display_name, + "enabled": u.enabled, + "access_keys": u.access_keys.iter().map(|k| { + serde_json::json!({ + "access_key": k.access_key, + "status": k.status, + "created_at": k.created_at, + }) + }).collect::>(), + "policy_count": u.policies.len(), + }) + }) + .collect() + } + + pub async fn get_user(&self, identifier: &str) -> Option { + self.reload_if_needed(); + let state = self.state.read(); + + let user = state + .user_records + .get(identifier) + .or_else(|| { + state.key_index.get(identifier).and_then(|uid| state.user_records.get(uid)) + })?; + + Some(serde_json::json!({ + "user_id": user.user_id, + "display_name": user.display_name, + "enabled": user.enabled, + "expires_at": user.expires_at, + "access_keys": user.access_keys.iter().map(|k| { + serde_json::json!({ + "access_key": k.access_key, + "status": k.status, + "created_at": k.created_at, + }) + }).collect::>(), + "policies": user.policies, + })) + } + + pub async fn set_user_enabled(&self, identifier: &str, enabled: bool) -> Result<(), String> { + let content = std::fs::read_to_string(&self.config_path) + .map_err(|e| format!("Failed to read IAM config: {}", e))?; + + let mut config: IamConfig = serde_json::from_str(&content) + .map_err(|e| format!("Failed to parse IAM config: {}", e))?; + + let user = config + .users + .iter_mut() + .find(|u| { + u.user_id == identifier + || u.access_keys.iter().any(|k| k.access_key == identifier) + }) + .ok_or_else(|| "User not found".to_string())?; + + user.enabled = enabled; + + let json = serde_json::to_string_pretty(&config) + .map_err(|e| format!("Failed to serialize IAM config: {}", e))?; + std::fs::write(&self.config_path, json) + .map_err(|e| format!("Failed to write IAM config: {}", e))?; + + self.reload(); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn test_iam_json() -> String { + serde_json::json!({ + "version": 2, + "users": [{ + "user_id": "u-test1234", + "display_name": "admin", + "enabled": true, + "access_keys": [{ + "access_key": "AKIAIOSFODNN7EXAMPLE", + "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "status": "active", + "created_at": "2024-01-01T00:00:00Z" + }], + "policies": [{ + "bucket": "*", + "actions": ["*"], + "prefix": "*" + }] + }] + }) + .to_string() + } + + #[test] + fn test_load_and_lookup() { + let mut tmp = tempfile::NamedTempFile::new().unwrap(); + tmp.write_all(test_iam_json().as_bytes()).unwrap(); + tmp.flush().unwrap(); + + let svc = IamService::new(tmp.path().to_path_buf()); + let secret = svc.get_secret_key("AKIAIOSFODNN7EXAMPLE"); + assert_eq!( + secret.unwrap(), + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + ); + } + + #[test] + fn test_get_principal() { + let mut tmp = tempfile::NamedTempFile::new().unwrap(); + tmp.write_all(test_iam_json().as_bytes()).unwrap(); + tmp.flush().unwrap(); + + let svc = IamService::new(tmp.path().to_path_buf()); + let principal = svc.get_principal("AKIAIOSFODNN7EXAMPLE").unwrap(); + assert_eq!(principal.display_name, "admin"); + assert_eq!(principal.user_id, "u-test1234"); + assert!(principal.is_admin); + } + + #[test] + fn test_authenticate_success() { + let mut tmp = tempfile::NamedTempFile::new().unwrap(); + tmp.write_all(test_iam_json().as_bytes()).unwrap(); + tmp.flush().unwrap(); + + let svc = IamService::new(tmp.path().to_path_buf()); + let principal = svc + .authenticate( + "AKIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + ) + .unwrap(); + assert_eq!(principal.display_name, "admin"); + } + + #[test] + fn test_authenticate_wrong_secret() { + let mut tmp = tempfile::NamedTempFile::new().unwrap(); + tmp.write_all(test_iam_json().as_bytes()).unwrap(); + tmp.flush().unwrap(); + + let svc = IamService::new(tmp.path().to_path_buf()); + assert!(svc.authenticate("AKIAIOSFODNN7EXAMPLE", "wrongsecret").is_none()); + } + + #[test] + fn test_unknown_key_returns_none() { + let mut tmp = tempfile::NamedTempFile::new().unwrap(); + tmp.write_all(test_iam_json().as_bytes()).unwrap(); + tmp.flush().unwrap(); + + let svc = IamService::new(tmp.path().to_path_buf()); + assert!(svc.get_secret_key("NONEXISTENTKEY").is_none()); + assert!(svc.get_principal("NONEXISTENTKEY").is_none()); + } + + #[test] + fn test_disabled_user() { + let json = serde_json::json!({ + "version": 2, + "users": [{ + "user_id": "u-disabled", + "display_name": "disabled-user", + "enabled": false, + "access_keys": [{ + "access_key": "DISABLED_KEY", + "secret_key": "secret123", + "status": "active" + }], + "policies": [] + }] + }) + .to_string(); + + let mut tmp = tempfile::NamedTempFile::new().unwrap(); + tmp.write_all(json.as_bytes()).unwrap(); + tmp.flush().unwrap(); + + let svc = IamService::new(tmp.path().to_path_buf()); + assert!(svc.get_secret_key("DISABLED_KEY").is_none()); + } + + #[test] + fn test_inactive_key() { + let json = serde_json::json!({ + "version": 2, + "users": [{ + "user_id": "u-test", + "display_name": "test", + "enabled": true, + "access_keys": [{ + "access_key": "INACTIVE_KEY", + "secret_key": "secret123", + "status": "inactive" + }], + "policies": [] + }] + }) + .to_string(); + + let mut tmp = tempfile::NamedTempFile::new().unwrap(); + tmp.write_all(json.as_bytes()).unwrap(); + tmp.flush().unwrap(); + + let svc = IamService::new(tmp.path().to_path_buf()); + assert!(svc.get_secret_key("INACTIVE_KEY").is_none()); + } +} diff --git a/myfsio-engine/crates/myfsio-auth/src/lib.rs b/myfsio-engine/crates/myfsio-auth/src/lib.rs new file mode 100644 index 0000000..083404f --- /dev/null +++ b/myfsio-engine/crates/myfsio-auth/src/lib.rs @@ -0,0 +1,4 @@ +pub mod sigv4; +pub mod principal; +pub mod iam; +mod fernet; diff --git a/myfsio-engine/crates/myfsio-auth/src/principal.rs b/myfsio-engine/crates/myfsio-auth/src/principal.rs new file mode 100644 index 0000000..52fe839 --- /dev/null +++ b/myfsio-engine/crates/myfsio-auth/src/principal.rs @@ -0,0 +1 @@ +pub use myfsio_common::types::Principal; diff --git a/myfsio-engine/crates/myfsio-auth/src/sigv4.rs b/myfsio-engine/crates/myfsio-auth/src/sigv4.rs new file mode 100644 index 0000000..3fd50c7 --- /dev/null +++ b/myfsio-engine/crates/myfsio-auth/src/sigv4.rs @@ -0,0 +1,258 @@ +use hmac::{Hmac, Mac}; +use lru::LruCache; +use parking_lot::Mutex; +use percent_encoding::{percent_encode, AsciiSet, NON_ALPHANUMERIC}; +use sha2::{Digest, Sha256}; +use std::num::NonZeroUsize; +use std::sync::LazyLock; +use std::time::Instant; + +type HmacSha256 = Hmac; + +struct CacheEntry { + key: Vec, + created: Instant, +} + +static SIGNING_KEY_CACHE: LazyLock>> = + LazyLock::new(|| Mutex::new(LruCache::new(NonZeroUsize::new(256).unwrap()))); + +const CACHE_TTL_SECS: u64 = 60; + +const AWS_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC + .remove(b'-') + .remove(b'_') + .remove(b'.') + .remove(b'~'); + +fn hmac_sha256(key: &[u8], msg: &[u8]) -> Vec { + let mut mac = HmacSha256::new_from_slice(key).expect("HMAC key length is always valid"); + mac.update(msg); + mac.finalize().into_bytes().to_vec() +} + +fn sha256_hex(data: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(data); + hex::encode(hasher.finalize()) +} + +fn aws_uri_encode(input: &str) -> String { + percent_encode(input.as_bytes(), AWS_ENCODE_SET).to_string() +} + +pub fn derive_signing_key_cached( + secret_key: &str, + date_stamp: &str, + region: &str, + service: &str, +) -> Vec { + let cache_key = ( + secret_key.to_owned(), + date_stamp.to_owned(), + region.to_owned(), + service.to_owned(), + ); + + { + let mut cache = SIGNING_KEY_CACHE.lock(); + if let Some(entry) = cache.get(&cache_key) { + if entry.created.elapsed().as_secs() < CACHE_TTL_SECS { + return entry.key.clone(); + } + cache.pop(&cache_key); + } + } + + let k_date = hmac_sha256(format!("AWS4{}", secret_key).as_bytes(), date_stamp.as_bytes()); + let k_region = hmac_sha256(&k_date, region.as_bytes()); + let k_service = hmac_sha256(&k_region, service.as_bytes()); + let k_signing = hmac_sha256(&k_service, b"aws4_request"); + + { + let mut cache = SIGNING_KEY_CACHE.lock(); + cache.put( + cache_key, + CacheEntry { + key: k_signing.clone(), + created: Instant::now(), + }, + ); + } + + k_signing +} + +fn constant_time_compare_inner(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + let mut result: u8 = 0; + for (x, y) in a.iter().zip(b.iter()) { + result |= x ^ y; + } + result == 0 +} + +pub fn verify_sigv4_signature( + method: &str, + canonical_uri: &str, + query_params: &[(String, String)], + signed_headers_str: &str, + header_values: &[(String, String)], + payload_hash: &str, + amz_date: &str, + date_stamp: &str, + region: &str, + service: &str, + secret_key: &str, + provided_signature: &str, +) -> bool { + let mut sorted_params = query_params.to_vec(); + sorted_params.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1))); + + let canonical_query_string = sorted_params + .iter() + .map(|(k, v)| format!("{}={}", aws_uri_encode(k), aws_uri_encode(v))) + .collect::>() + .join("&"); + + let mut canonical_headers = String::new(); + for (name, value) in header_values { + let lower_name = name.to_lowercase(); + let normalized = value.split_whitespace().collect::>().join(" "); + let final_value = if lower_name == "expect" && normalized.is_empty() { + "100-continue" + } else { + &normalized + }; + canonical_headers.push_str(&lower_name); + canonical_headers.push(':'); + canonical_headers.push_str(final_value); + canonical_headers.push('\n'); + } + + let canonical_request = format!( + "{}\n{}\n{}\n{}\n{}\n{}", + method, canonical_uri, canonical_query_string, canonical_headers, signed_headers_str, + payload_hash + ); + + let credential_scope = format!("{}/{}/{}/aws4_request", date_stamp, region, service); + let cr_hash = sha256_hex(canonical_request.as_bytes()); + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + amz_date, credential_scope, cr_hash + ); + + let signing_key = derive_signing_key_cached(secret_key, date_stamp, region, service); + let calculated = hmac_sha256(&signing_key, string_to_sign.as_bytes()); + let calculated_hex = hex::encode(&calculated); + + constant_time_compare_inner(calculated_hex.as_bytes(), provided_signature.as_bytes()) +} + +pub fn derive_signing_key( + secret_key: &str, + date_stamp: &str, + region: &str, + service: &str, +) -> Vec { + derive_signing_key_cached(secret_key, date_stamp, region, service) +} + +pub fn compute_signature(signing_key: &[u8], string_to_sign: &str) -> String { + let sig = hmac_sha256(signing_key, string_to_sign.as_bytes()); + hex::encode(sig) +} + +pub fn build_string_to_sign( + amz_date: &str, + credential_scope: &str, + canonical_request: &str, +) -> String { + let cr_hash = sha256_hex(canonical_request.as_bytes()); + format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + amz_date, credential_scope, cr_hash + ) +} + +pub fn constant_time_compare(a: &str, b: &str) -> bool { + constant_time_compare_inner(a.as_bytes(), b.as_bytes()) +} + +pub fn clear_signing_key_cache() { + SIGNING_KEY_CACHE.lock().clear(); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_derive_signing_key() { + let key = derive_signing_key("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "20130524", "us-east-1", "s3"); + assert_eq!(key.len(), 32); + } + + #[test] + fn test_derive_signing_key_cached() { + let key1 = derive_signing_key("secret", "20240101", "us-east-1", "s3"); + let key2 = derive_signing_key("secret", "20240101", "us-east-1", "s3"); + assert_eq!(key1, key2); + } + + #[test] + fn test_constant_time_compare() { + assert!(constant_time_compare("abc", "abc")); + assert!(!constant_time_compare("abc", "abd")); + assert!(!constant_time_compare("abc", "abcd")); + } + + #[test] + fn test_build_string_to_sign() { + let result = build_string_to_sign("20130524T000000Z", "20130524/us-east-1/s3/aws4_request", "GET\n/\n\nhost:example.com\n\nhost\nUNSIGNED-PAYLOAD"); + assert!(result.starts_with("AWS4-HMAC-SHA256\n")); + assert!(result.contains("20130524T000000Z")); + } + + #[test] + fn test_aws_uri_encode() { + assert_eq!(aws_uri_encode("hello world"), "hello%20world"); + assert_eq!(aws_uri_encode("test-file_name.txt"), "test-file_name.txt"); + assert_eq!(aws_uri_encode("a/b"), "a%2Fb"); + } + + #[test] + fn test_verify_sigv4_roundtrip() { + let secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + let date_stamp = "20130524"; + let region = "us-east-1"; + let service = "s3"; + let amz_date = "20130524T000000Z"; + + let signing_key = derive_signing_key(secret, date_stamp, region, service); + + let canonical_request = "GET\n/\n\nhost:examplebucket.s3.amazonaws.com\n\nhost\nUNSIGNED-PAYLOAD"; + let string_to_sign = build_string_to_sign(amz_date, &format!("{}/{}/{}/aws4_request", date_stamp, region, service), canonical_request); + + let signature = compute_signature(&signing_key, &string_to_sign); + + let result = verify_sigv4_signature( + "GET", + "/", + &[], + "host", + &[("host".to_string(), "examplebucket.s3.amazonaws.com".to_string())], + "UNSIGNED-PAYLOAD", + amz_date, + date_stamp, + region, + service, + secret, + &signature, + ); + assert!(result); + } +} diff --git a/myfsio-engine/crates/myfsio-common/Cargo.toml b/myfsio-engine/crates/myfsio-common/Cargo.toml new file mode 100644 index 0000000..e140021 --- /dev/null +++ b/myfsio-engine/crates/myfsio-common/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "myfsio-common" +version = "0.1.0" +edition = "2021" + +[dependencies] +thiserror = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +chrono = { workspace = true } +uuid = { workspace = true } diff --git a/myfsio-engine/crates/myfsio-common/src/constants.rs b/myfsio-engine/crates/myfsio-common/src/constants.rs new file mode 100644 index 0000000..9a9e25c --- /dev/null +++ b/myfsio-engine/crates/myfsio-common/src/constants.rs @@ -0,0 +1,20 @@ +pub const SYSTEM_ROOT: &str = ".myfsio.sys"; +pub const SYSTEM_BUCKETS_DIR: &str = "buckets"; +pub const SYSTEM_MULTIPART_DIR: &str = "multipart"; +pub const BUCKET_META_DIR: &str = "meta"; +pub const BUCKET_VERSIONS_DIR: &str = "versions"; +pub const BUCKET_CONFIG_FILE: &str = ".bucket.json"; +pub const STATS_FILE: &str = "stats.json"; +pub const ETAG_INDEX_FILE: &str = "etag_index.json"; +pub const INDEX_FILE: &str = "_index.json"; +pub const MANIFEST_FILE: &str = "manifest.json"; + +pub const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"]; + +pub const DEFAULT_REGION: &str = "us-east-1"; +pub const AWS_SERVICE: &str = "s3"; + +pub const DEFAULT_MAX_KEYS: usize = 1000; +pub const DEFAULT_OBJECT_KEY_MAX_BYTES: usize = 1024; +pub const DEFAULT_CHUNK_SIZE: usize = 65536; +pub const STREAM_CHUNK_SIZE: usize = 1_048_576; diff --git a/myfsio-engine/crates/myfsio-common/src/error.rs b/myfsio-engine/crates/myfsio-common/src/error.rs new file mode 100644 index 0000000..ae115f6 --- /dev/null +++ b/myfsio-engine/crates/myfsio-common/src/error.rs @@ -0,0 +1,221 @@ +use std::fmt; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum S3ErrorCode { + AccessDenied, + BucketAlreadyExists, + BucketNotEmpty, + EntityTooLarge, + InternalError, + InvalidAccessKeyId, + InvalidArgument, + InvalidBucketName, + InvalidKey, + InvalidRange, + InvalidRequest, + MalformedXML, + MethodNotAllowed, + NoSuchBucket, + NoSuchKey, + NoSuchUpload, + NoSuchVersion, + NoSuchTagSet, + PreconditionFailed, + NotModified, + QuotaExceeded, + SignatureDoesNotMatch, + SlowDown, +} + +impl S3ErrorCode { + pub fn http_status(&self) -> u16 { + match self { + Self::AccessDenied => 403, + Self::BucketAlreadyExists => 409, + Self::BucketNotEmpty => 409, + Self::EntityTooLarge => 413, + Self::InternalError => 500, + Self::InvalidAccessKeyId => 403, + Self::InvalidArgument => 400, + Self::InvalidBucketName => 400, + Self::InvalidKey => 400, + Self::InvalidRange => 416, + Self::InvalidRequest => 400, + Self::MalformedXML => 400, + Self::MethodNotAllowed => 405, + Self::NoSuchBucket => 404, + Self::NoSuchKey => 404, + Self::NoSuchUpload => 404, + Self::NoSuchVersion => 404, + Self::NoSuchTagSet => 404, + Self::PreconditionFailed => 412, + Self::NotModified => 304, + Self::QuotaExceeded => 403, + Self::SignatureDoesNotMatch => 403, + Self::SlowDown => 429, + } + } + + pub fn as_str(&self) -> &'static str { + match self { + Self::AccessDenied => "AccessDenied", + Self::BucketAlreadyExists => "BucketAlreadyExists", + Self::BucketNotEmpty => "BucketNotEmpty", + Self::EntityTooLarge => "EntityTooLarge", + Self::InternalError => "InternalError", + Self::InvalidAccessKeyId => "InvalidAccessKeyId", + Self::InvalidArgument => "InvalidArgument", + Self::InvalidBucketName => "InvalidBucketName", + Self::InvalidKey => "InvalidKey", + Self::InvalidRange => "InvalidRange", + Self::InvalidRequest => "InvalidRequest", + Self::MalformedXML => "MalformedXML", + Self::MethodNotAllowed => "MethodNotAllowed", + Self::NoSuchBucket => "NoSuchBucket", + Self::NoSuchKey => "NoSuchKey", + Self::NoSuchUpload => "NoSuchUpload", + Self::NoSuchVersion => "NoSuchVersion", + Self::NoSuchTagSet => "NoSuchTagSet", + Self::PreconditionFailed => "PreconditionFailed", + Self::NotModified => "NotModified", + Self::QuotaExceeded => "QuotaExceeded", + Self::SignatureDoesNotMatch => "SignatureDoesNotMatch", + Self::SlowDown => "SlowDown", + } + } + + pub fn default_message(&self) -> &'static str { + match self { + Self::AccessDenied => "Access Denied", + Self::BucketAlreadyExists => "The requested bucket name is not available", + Self::BucketNotEmpty => "The bucket you tried to delete is not empty", + Self::EntityTooLarge => "Your proposed upload exceeds the maximum allowed size", + Self::InternalError => "We encountered an internal error. Please try again.", + Self::InvalidAccessKeyId => "The access key ID you provided does not exist", + Self::InvalidArgument => "Invalid argument", + Self::InvalidBucketName => "The specified bucket is not valid", + Self::InvalidKey => "The specified key is not valid", + Self::InvalidRange => "The requested range is not satisfiable", + Self::InvalidRequest => "Invalid request", + Self::MalformedXML => "The XML you provided was not well-formed", + Self::MethodNotAllowed => "The specified method is not allowed against this resource", + Self::NoSuchBucket => "The specified bucket does not exist", + Self::NoSuchKey => "The specified key does not exist", + Self::NoSuchUpload => "The specified multipart upload does not exist", + Self::NoSuchVersion => "The specified version does not exist", + Self::NoSuchTagSet => "The TagSet does not exist", + Self::PreconditionFailed => "At least one of the preconditions you specified did not hold", + Self::NotModified => "Not Modified", + Self::QuotaExceeded => "The bucket quota has been exceeded", + Self::SignatureDoesNotMatch => "The request signature we calculated does not match the signature you provided", + Self::SlowDown => "Please reduce your request rate", + } + } +} + +impl fmt::Display for S3ErrorCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Debug, Clone)] +pub struct S3Error { + pub code: S3ErrorCode, + pub message: String, + pub resource: String, + pub request_id: String, +} + +impl S3Error { + pub fn new(code: S3ErrorCode, message: impl Into) -> Self { + Self { + code, + message: message.into(), + resource: String::new(), + request_id: String::new(), + } + } + + pub fn from_code(code: S3ErrorCode) -> Self { + Self::new(code, code.default_message()) + } + + pub fn with_resource(mut self, resource: impl Into) -> Self { + self.resource = resource.into(); + self + } + + pub fn with_request_id(mut self, request_id: impl Into) -> Self { + self.request_id = request_id.into(); + self + } + + pub fn http_status(&self) -> u16 { + self.code.http_status() + } + + pub fn to_xml(&self) -> String { + format!( + "\ + \ + {}\ + {}\ + {}\ + {}\ + ", + self.code.as_str(), + xml_escape(&self.message), + xml_escape(&self.resource), + xml_escape(&self.request_id), + ) + } +} + +impl fmt::Display for S3Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}: {}", self.code, self.message) + } +} + +impl std::error::Error for S3Error {} + +fn xml_escape(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_error_codes() { + assert_eq!(S3ErrorCode::NoSuchKey.http_status(), 404); + assert_eq!(S3ErrorCode::AccessDenied.http_status(), 403); + assert_eq!(S3ErrorCode::NoSuchBucket.as_str(), "NoSuchBucket"); + } + + #[test] + fn test_error_to_xml() { + let err = S3Error::from_code(S3ErrorCode::NoSuchKey) + .with_resource("/test-bucket/test-key") + .with_request_id("abc123"); + let xml = err.to_xml(); + assert!(xml.contains("NoSuchKey")); + assert!(xml.contains("/test-bucket/test-key")); + assert!(xml.contains("abc123")); + } + + #[test] + fn test_xml_escape() { + let err = S3Error::new(S3ErrorCode::InvalidArgument, "key & \"value\"") + .with_resource("/bucket/key&"); + let xml = err.to_xml(); + assert!(xml.contains("<test>")); + assert!(xml.contains("&")); + } +} diff --git a/myfsio-engine/crates/myfsio-common/src/lib.rs b/myfsio-engine/crates/myfsio-common/src/lib.rs new file mode 100644 index 0000000..ad67240 --- /dev/null +++ b/myfsio-engine/crates/myfsio-common/src/lib.rs @@ -0,0 +1,3 @@ +pub mod constants; +pub mod error; +pub mod types; diff --git a/myfsio-engine/crates/myfsio-common/src/types.rs b/myfsio-engine/crates/myfsio-common/src/types.rs new file mode 100644 index 0000000..73f6aee --- /dev/null +++ b/myfsio-engine/crates/myfsio-common/src/types.rs @@ -0,0 +1,172 @@ +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ObjectMeta { + pub key: String, + pub size: u64, + pub last_modified: DateTime, + pub etag: Option, + pub content_type: Option, + pub storage_class: Option, + pub metadata: HashMap, +} + +impl ObjectMeta { + pub fn new(key: String, size: u64, last_modified: DateTime) -> Self { + Self { + key, + size, + last_modified, + etag: None, + content_type: None, + storage_class: Some("STANDARD".to_string()), + metadata: HashMap::new(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BucketMeta { + pub name: String, + pub creation_date: DateTime, +} + +#[derive(Debug, Clone, Default)] +pub struct BucketStats { + pub objects: u64, + pub bytes: u64, + pub version_count: u64, + pub version_bytes: u64, +} + +impl BucketStats { + pub fn total_objects(&self) -> u64 { + self.objects + self.version_count + } + + pub fn total_bytes(&self) -> u64 { + self.bytes + self.version_bytes + } +} + +#[derive(Debug, Clone)] +pub struct ListObjectsResult { + pub objects: Vec, + pub is_truncated: bool, + pub next_continuation_token: Option, +} + +#[derive(Debug, Clone)] +pub struct ShallowListResult { + pub objects: Vec, + pub common_prefixes: Vec, + pub is_truncated: bool, + pub next_continuation_token: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct ListParams { + pub max_keys: usize, + pub continuation_token: Option, + pub prefix: Option, + pub start_after: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct ShallowListParams { + pub prefix: String, + pub delimiter: String, + pub max_keys: usize, + pub continuation_token: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PartMeta { + pub part_number: u32, + pub etag: String, + pub size: u64, + pub last_modified: Option>, +} + +#[derive(Debug, Clone)] +pub struct PartInfo { + pub part_number: u32, + pub etag: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MultipartUploadInfo { + pub upload_id: String, + pub key: String, + pub initiated: DateTime, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VersionInfo { + pub version_id: String, + pub key: String, + pub size: u64, + pub last_modified: DateTime, + pub etag: Option, + pub is_latest: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Tag { + pub key: String, + pub value: String, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct BucketConfig { + #[serde(default)] + pub versioning_enabled: bool, + #[serde(default)] + pub tags: Vec, + #[serde(default)] + pub cors: Option, + #[serde(default)] + pub encryption: Option, + #[serde(default)] + pub lifecycle: Option, + #[serde(default)] + pub website: Option, + #[serde(default)] + pub quota: Option, + #[serde(default)] + pub acl: Option, + #[serde(default)] + pub notification: Option, + #[serde(default)] + pub logging: Option, + #[serde(default)] + pub object_lock: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QuotaConfig { + pub max_bytes: Option, + pub max_objects: Option, +} + +#[derive(Debug, Clone)] +pub struct Principal { + pub access_key: String, + pub user_id: String, + pub display_name: String, + pub is_admin: bool, +} + +impl Principal { + pub fn new(access_key: String, user_id: String, display_name: String, is_admin: bool) -> Self { + Self { + access_key, + user_id, + display_name, + is_admin, + } + } +} diff --git a/myfsio-engine/crates/myfsio-crypto/Cargo.toml b/myfsio-engine/crates/myfsio-crypto/Cargo.toml new file mode 100644 index 0000000..45cecc7 --- /dev/null +++ b/myfsio-engine/crates/myfsio-crypto/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "myfsio-crypto" +version = "0.1.0" +edition = "2021" + +[dependencies] +myfsio-common = { path = "../myfsio-common" } +md-5 = { workspace = true } +sha2 = { workspace = true } +hex = { workspace = true } +aes-gcm = { workspace = true } +hkdf = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +uuid = { workspace = true } +chrono = { workspace = true } +base64 = { workspace = true } +rand = "0.8" + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tempfile = "3" diff --git a/myfsio-engine/crates/myfsio-crypto/src/aes_gcm.rs b/myfsio-engine/crates/myfsio-crypto/src/aes_gcm.rs new file mode 100644 index 0000000..dc7a346 --- /dev/null +++ b/myfsio-engine/crates/myfsio-crypto/src/aes_gcm.rs @@ -0,0 +1,238 @@ +use aes_gcm::aead::Aead; +use aes_gcm::{Aes256Gcm, KeyInit, Nonce}; +use hkdf::Hkdf; +use sha2::Sha256; +use std::fs::File; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::Path; +use thiserror::Error; + +const DEFAULT_CHUNK_SIZE: usize = 65536; +const HEADER_SIZE: usize = 4; + +#[derive(Debug, Error)] +pub enum CryptoError { + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + #[error("Invalid key size: expected 32 bytes, got {0}")] + InvalidKeySize(usize), + #[error("Invalid nonce size: expected 12 bytes, got {0}")] + InvalidNonceSize(usize), + #[error("Encryption failed: {0}")] + EncryptionFailed(String), + #[error("Decryption failed at chunk {0}")] + DecryptionFailed(u32), + #[error("HKDF expand failed: {0}")] + HkdfFailed(String), +} + +fn read_exact_chunk(reader: &mut impl Read, buf: &mut [u8]) -> std::io::Result { + let mut filled = 0; + while filled < buf.len() { + match reader.read(&mut buf[filled..]) { + Ok(0) => break, + Ok(n) => filled += n, + Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + } + } + Ok(filled) +} + +fn derive_chunk_nonce(base_nonce: &[u8], chunk_index: u32) -> Result<[u8; 12], CryptoError> { + let hkdf = Hkdf::::new(Some(base_nonce), b"chunk_nonce"); + let mut okm = [0u8; 12]; + hkdf.expand(&chunk_index.to_be_bytes(), &mut okm) + .map_err(|e| CryptoError::HkdfFailed(e.to_string()))?; + Ok(okm) +} + +pub fn encrypt_stream_chunked( + input_path: &Path, + output_path: &Path, + key: &[u8], + base_nonce: &[u8], + chunk_size: Option, +) -> Result { + if key.len() != 32 { + return Err(CryptoError::InvalidKeySize(key.len())); + } + if base_nonce.len() != 12 { + return Err(CryptoError::InvalidNonceSize(base_nonce.len())); + } + + let chunk_size = chunk_size.unwrap_or(DEFAULT_CHUNK_SIZE); + let key_arr: [u8; 32] = key.try_into().unwrap(); + let nonce_arr: [u8; 12] = base_nonce.try_into().unwrap(); + let cipher = Aes256Gcm::new(&key_arr.into()); + + let mut infile = File::open(input_path)?; + let mut outfile = File::create(output_path)?; + + outfile.write_all(&[0u8; 4])?; + + let mut buf = vec![0u8; chunk_size]; + let mut chunk_index: u32 = 0; + + loop { + let n = read_exact_chunk(&mut infile, &mut buf)?; + if n == 0 { + break; + } + + let nonce_bytes = derive_chunk_nonce(&nonce_arr, chunk_index)?; + let nonce = Nonce::from_slice(&nonce_bytes); + + let encrypted = cipher + .encrypt(nonce, &buf[..n]) + .map_err(|e| CryptoError::EncryptionFailed(e.to_string()))?; + + let size = encrypted.len() as u32; + outfile.write_all(&size.to_be_bytes())?; + outfile.write_all(&encrypted)?; + + chunk_index += 1; + } + + outfile.seek(SeekFrom::Start(0))?; + outfile.write_all(&chunk_index.to_be_bytes())?; + + Ok(chunk_index) +} + +pub fn decrypt_stream_chunked( + input_path: &Path, + output_path: &Path, + key: &[u8], + base_nonce: &[u8], +) -> Result { + if key.len() != 32 { + return Err(CryptoError::InvalidKeySize(key.len())); + } + if base_nonce.len() != 12 { + return Err(CryptoError::InvalidNonceSize(base_nonce.len())); + } + + let key_arr: [u8; 32] = key.try_into().unwrap(); + let nonce_arr: [u8; 12] = base_nonce.try_into().unwrap(); + let cipher = Aes256Gcm::new(&key_arr.into()); + + let mut infile = File::open(input_path)?; + let mut outfile = File::create(output_path)?; + + let mut header = [0u8; HEADER_SIZE]; + infile.read_exact(&mut header)?; + let chunk_count = u32::from_be_bytes(header); + + let mut size_buf = [0u8; HEADER_SIZE]; + for chunk_index in 0..chunk_count { + infile.read_exact(&mut size_buf)?; + let chunk_size = u32::from_be_bytes(size_buf) as usize; + + let mut encrypted = vec![0u8; chunk_size]; + infile.read_exact(&mut encrypted)?; + + let nonce_bytes = derive_chunk_nonce(&nonce_arr, chunk_index)?; + let nonce = Nonce::from_slice(&nonce_bytes); + + let decrypted = cipher + .decrypt(nonce, encrypted.as_ref()) + .map_err(|_| CryptoError::DecryptionFailed(chunk_index))?; + + outfile.write_all(&decrypted)?; + } + + Ok(chunk_count) +} + +pub async fn encrypt_stream_chunked_async( + input_path: &Path, + output_path: &Path, + key: &[u8], + base_nonce: &[u8], + chunk_size: Option, +) -> Result { + let input_path = input_path.to_owned(); + let output_path = output_path.to_owned(); + let key = key.to_vec(); + let base_nonce = base_nonce.to_vec(); + tokio::task::spawn_blocking(move || { + encrypt_stream_chunked(&input_path, &output_path, &key, &base_nonce, chunk_size) + }) + .await + .map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))? +} + +pub async fn decrypt_stream_chunked_async( + input_path: &Path, + output_path: &Path, + key: &[u8], + base_nonce: &[u8], +) -> Result { + let input_path = input_path.to_owned(); + let output_path = output_path.to_owned(); + let key = key.to_vec(); + let base_nonce = base_nonce.to_vec(); + tokio::task::spawn_blocking(move || { + decrypt_stream_chunked(&input_path, &output_path, &key, &base_nonce) + }) + .await + .map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))? +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write as IoWrite; + + #[test] + fn test_encrypt_decrypt_roundtrip() { + let dir = tempfile::tempdir().unwrap(); + let input = dir.path().join("input.bin"); + let encrypted = dir.path().join("encrypted.bin"); + let decrypted = dir.path().join("decrypted.bin"); + + let data = b"Hello, this is a test of AES-256-GCM chunked encryption!"; + std::fs::File::create(&input).unwrap().write_all(data).unwrap(); + + let key = [0x42u8; 32]; + let nonce = [0x01u8; 12]; + + let chunks = encrypt_stream_chunked(&input, &encrypted, &key, &nonce, Some(16)).unwrap(); + assert!(chunks > 0); + + let chunks2 = decrypt_stream_chunked(&encrypted, &decrypted, &key, &nonce).unwrap(); + assert_eq!(chunks, chunks2); + + let result = std::fs::read(&decrypted).unwrap(); + assert_eq!(result, data); + } + + #[test] + fn test_invalid_key_size() { + let dir = tempfile::tempdir().unwrap(); + let input = dir.path().join("input.bin"); + std::fs::File::create(&input).unwrap().write_all(b"test").unwrap(); + + let result = encrypt_stream_chunked(&input, &dir.path().join("out"), &[0u8; 16], &[0u8; 12], None); + assert!(matches!(result, Err(CryptoError::InvalidKeySize(16)))); + } + + #[test] + fn test_wrong_key_fails_decrypt() { + let dir = tempfile::tempdir().unwrap(); + let input = dir.path().join("input.bin"); + let encrypted = dir.path().join("encrypted.bin"); + let decrypted = dir.path().join("decrypted.bin"); + + std::fs::File::create(&input).unwrap().write_all(b"secret data").unwrap(); + + let key = [0x42u8; 32]; + let nonce = [0x01u8; 12]; + encrypt_stream_chunked(&input, &encrypted, &key, &nonce, None).unwrap(); + + let wrong_key = [0x43u8; 32]; + let result = decrypt_stream_chunked(&encrypted, &decrypted, &wrong_key, &nonce); + assert!(matches!(result, Err(CryptoError::DecryptionFailed(_)))); + } +} diff --git a/myfsio-engine/crates/myfsio-crypto/src/encryption.rs b/myfsio-engine/crates/myfsio-crypto/src/encryption.rs new file mode 100644 index 0000000..2a78f3c --- /dev/null +++ b/myfsio-engine/crates/myfsio-crypto/src/encryption.rs @@ -0,0 +1,375 @@ +use base64::engine::general_purpose::STANDARD as B64; +use base64::Engine; +use rand::RngCore; +use std::collections::HashMap; +use std::path::Path; + +use crate::aes_gcm::{ + encrypt_stream_chunked, decrypt_stream_chunked, CryptoError, +}; +use crate::kms::KmsService; + +#[derive(Debug, Clone, PartialEq)] +pub enum SseAlgorithm { + Aes256, + AwsKms, + CustomerProvided, +} + +impl SseAlgorithm { + pub fn as_str(&self) -> &'static str { + match self { + SseAlgorithm::Aes256 => "AES256", + SseAlgorithm::AwsKms => "aws:kms", + SseAlgorithm::CustomerProvided => "AES256", + } + } +} + +#[derive(Debug, Clone)] +pub struct EncryptionContext { + pub algorithm: SseAlgorithm, + pub kms_key_id: Option, + pub customer_key: Option>, +} + +#[derive(Debug, Clone)] +pub struct EncryptionMetadata { + pub algorithm: String, + pub nonce: String, + pub encrypted_data_key: Option, + pub kms_key_id: Option, +} + +impl EncryptionMetadata { + pub fn to_metadata_map(&self) -> HashMap { + let mut map = HashMap::new(); + map.insert( + "x-amz-server-side-encryption".to_string(), + self.algorithm.clone(), + ); + map.insert("x-amz-encryption-nonce".to_string(), self.nonce.clone()); + if let Some(ref dk) = self.encrypted_data_key { + map.insert("x-amz-encrypted-data-key".to_string(), dk.clone()); + } + if let Some(ref kid) = self.kms_key_id { + map.insert("x-amz-encryption-key-id".to_string(), kid.clone()); + } + map + } + + pub fn from_metadata(meta: &HashMap) -> Option { + let algorithm = meta.get("x-amz-server-side-encryption")?; + let nonce = meta.get("x-amz-encryption-nonce")?; + Some(Self { + algorithm: algorithm.clone(), + nonce: nonce.clone(), + encrypted_data_key: meta.get("x-amz-encrypted-data-key").cloned(), + kms_key_id: meta.get("x-amz-encryption-key-id").cloned(), + }) + } + + pub fn is_encrypted(meta: &HashMap) -> bool { + meta.contains_key("x-amz-server-side-encryption") + } + + pub fn clean_metadata(meta: &mut HashMap) { + meta.remove("x-amz-server-side-encryption"); + meta.remove("x-amz-encryption-nonce"); + meta.remove("x-amz-encrypted-data-key"); + meta.remove("x-amz-encryption-key-id"); + } +} + +pub struct EncryptionService { + master_key: [u8; 32], + kms: Option>, +} + +impl EncryptionService { + pub fn new(master_key: [u8; 32], kms: Option>) -> Self { + Self { master_key, kms } + } + + pub fn generate_data_key(&self) -> ([u8; 32], [u8; 12]) { + let mut data_key = [0u8; 32]; + let mut nonce = [0u8; 12]; + rand::thread_rng().fill_bytes(&mut data_key); + rand::thread_rng().fill_bytes(&mut nonce); + (data_key, nonce) + } + + pub fn wrap_data_key(&self, data_key: &[u8; 32]) -> Result { + use aes_gcm::aead::Aead; + use aes_gcm::{Aes256Gcm, KeyInit, Nonce}; + + let cipher = Aes256Gcm::new((&self.master_key).into()); + let mut nonce_bytes = [0u8; 12]; + rand::thread_rng().fill_bytes(&mut nonce_bytes); + let nonce = Nonce::from_slice(&nonce_bytes); + + let encrypted = cipher + .encrypt(nonce, data_key.as_slice()) + .map_err(|e| CryptoError::EncryptionFailed(e.to_string()))?; + + let mut combined = Vec::with_capacity(12 + encrypted.len()); + combined.extend_from_slice(&nonce_bytes); + combined.extend_from_slice(&encrypted); + Ok(B64.encode(&combined)) + } + + pub fn unwrap_data_key(&self, wrapped_b64: &str) -> Result<[u8; 32], CryptoError> { + use aes_gcm::aead::Aead; + use aes_gcm::{Aes256Gcm, KeyInit, Nonce}; + + let combined = B64.decode(wrapped_b64).map_err(|e| { + CryptoError::EncryptionFailed(format!("Bad wrapped key encoding: {}", e)) + })?; + if combined.len() < 12 { + return Err(CryptoError::EncryptionFailed( + "Wrapped key too short".to_string(), + )); + } + + let (nonce_bytes, ciphertext) = combined.split_at(12); + let cipher = Aes256Gcm::new((&self.master_key).into()); + let nonce = Nonce::from_slice(nonce_bytes); + + let plaintext = cipher + .decrypt(nonce, ciphertext) + .map_err(|_| CryptoError::DecryptionFailed(0))?; + + if plaintext.len() != 32 { + return Err(CryptoError::InvalidKeySize(plaintext.len())); + } + let mut key = [0u8; 32]; + key.copy_from_slice(&plaintext); + Ok(key) + } + + pub async fn encrypt_object( + &self, + input_path: &Path, + output_path: &Path, + ctx: &EncryptionContext, + ) -> Result { + let (data_key, nonce) = self.generate_data_key(); + + let (encrypted_data_key, kms_key_id) = match ctx.algorithm { + SseAlgorithm::Aes256 => { + let wrapped = self.wrap_data_key(&data_key)?; + (Some(wrapped), None) + } + SseAlgorithm::AwsKms => { + let kms = self + .kms + .as_ref() + .ok_or_else(|| CryptoError::EncryptionFailed("KMS not available".into()))?; + let kid = ctx + .kms_key_id + .as_ref() + .ok_or_else(|| CryptoError::EncryptionFailed("No KMS key ID".into()))?; + let ciphertext = kms.encrypt_data(kid, &data_key).await?; + (Some(B64.encode(&ciphertext)), Some(kid.clone())) + } + SseAlgorithm::CustomerProvided => { + (None, None) + } + }; + + let actual_key = if ctx.algorithm == SseAlgorithm::CustomerProvided { + let ck = ctx.customer_key.as_ref().ok_or_else(|| { + CryptoError::EncryptionFailed("No customer key provided".into()) + })?; + if ck.len() != 32 { + return Err(CryptoError::InvalidKeySize(ck.len())); + } + let mut k = [0u8; 32]; + k.copy_from_slice(ck); + k + } else { + data_key + }; + + let ip = input_path.to_owned(); + let op = output_path.to_owned(); + let ak = actual_key; + let n = nonce; + tokio::task::spawn_blocking(move || { + encrypt_stream_chunked(&ip, &op, &ak, &n, None) + }) + .await + .map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))??; + + Ok(EncryptionMetadata { + algorithm: ctx.algorithm.as_str().to_string(), + nonce: B64.encode(nonce), + encrypted_data_key, + kms_key_id, + }) + } + + pub async fn decrypt_object( + &self, + input_path: &Path, + output_path: &Path, + enc_meta: &EncryptionMetadata, + customer_key: Option<&[u8]>, + ) -> Result<(), CryptoError> { + let nonce_bytes = B64.decode(&enc_meta.nonce).map_err(|e| { + CryptoError::EncryptionFailed(format!("Bad nonce encoding: {}", e)) + })?; + if nonce_bytes.len() != 12 { + return Err(CryptoError::InvalidNonceSize(nonce_bytes.len())); + } + + let data_key: [u8; 32] = if let Some(ck) = customer_key { + if ck.len() != 32 { + return Err(CryptoError::InvalidKeySize(ck.len())); + } + let mut k = [0u8; 32]; + k.copy_from_slice(ck); + k + } else if enc_meta.algorithm == "aws:kms" { + let kms = self + .kms + .as_ref() + .ok_or_else(|| CryptoError::EncryptionFailed("KMS not available".into()))?; + let kid = enc_meta + .kms_key_id + .as_ref() + .ok_or_else(|| CryptoError::EncryptionFailed("No KMS key ID in metadata".into()))?; + let encrypted_dk = enc_meta.encrypted_data_key.as_ref().ok_or_else(|| { + CryptoError::EncryptionFailed("No encrypted data key in metadata".into()) + })?; + let ct = B64.decode(encrypted_dk).map_err(|e| { + CryptoError::EncryptionFailed(format!("Bad data key encoding: {}", e)) + })?; + let dk = kms.decrypt_data(kid, &ct).await?; + if dk.len() != 32 { + return Err(CryptoError::InvalidKeySize(dk.len())); + } + let mut k = [0u8; 32]; + k.copy_from_slice(&dk); + k + } else { + let wrapped = enc_meta.encrypted_data_key.as_ref().ok_or_else(|| { + CryptoError::EncryptionFailed("No encrypted data key in metadata".into()) + })?; + self.unwrap_data_key(wrapped)? + }; + + let ip = input_path.to_owned(); + let op = output_path.to_owned(); + let nb: [u8; 12] = nonce_bytes.try_into().unwrap(); + tokio::task::spawn_blocking(move || { + decrypt_stream_chunked(&ip, &op, &data_key, &nb) + }) + .await + .map_err(|e| CryptoError::Io(std::io::Error::new(std::io::ErrorKind::Other, e)))??; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn test_master_key() -> [u8; 32] { + [0x42u8; 32] + } + + #[test] + fn test_wrap_unwrap_data_key() { + let svc = EncryptionService::new(test_master_key(), None); + let dk = [0xAAu8; 32]; + let wrapped = svc.wrap_data_key(&dk).unwrap(); + let unwrapped = svc.unwrap_data_key(&wrapped).unwrap(); + assert_eq!(dk, unwrapped); + } + + #[tokio::test] + async fn test_encrypt_decrypt_object_sse_s3() { + let dir = tempfile::tempdir().unwrap(); + let input = dir.path().join("plain.bin"); + let encrypted = dir.path().join("enc.bin"); + let decrypted = dir.path().join("dec.bin"); + + let data = b"SSE-S3 encrypted content for testing!"; + std::fs::File::create(&input).unwrap().write_all(data).unwrap(); + + let svc = EncryptionService::new(test_master_key(), None); + + let ctx = EncryptionContext { + algorithm: SseAlgorithm::Aes256, + kms_key_id: None, + customer_key: None, + }; + + let meta = svc.encrypt_object(&input, &encrypted, &ctx).await.unwrap(); + assert_eq!(meta.algorithm, "AES256"); + assert!(meta.encrypted_data_key.is_some()); + + svc.decrypt_object(&encrypted, &decrypted, &meta, None) + .await + .unwrap(); + + let result = std::fs::read(&decrypted).unwrap(); + assert_eq!(result, data); + } + + #[tokio::test] + async fn test_encrypt_decrypt_object_sse_c() { + let dir = tempfile::tempdir().unwrap(); + let input = dir.path().join("plain.bin"); + let encrypted = dir.path().join("enc.bin"); + let decrypted = dir.path().join("dec.bin"); + + let data = b"SSE-C encrypted content!"; + std::fs::File::create(&input).unwrap().write_all(data).unwrap(); + + let customer_key = [0xBBu8; 32]; + let svc = EncryptionService::new(test_master_key(), None); + + let ctx = EncryptionContext { + algorithm: SseAlgorithm::CustomerProvided, + kms_key_id: None, + customer_key: Some(customer_key.to_vec()), + }; + + let meta = svc.encrypt_object(&input, &encrypted, &ctx).await.unwrap(); + assert!(meta.encrypted_data_key.is_none()); + + svc.decrypt_object(&encrypted, &decrypted, &meta, Some(&customer_key)) + .await + .unwrap(); + + let result = std::fs::read(&decrypted).unwrap(); + assert_eq!(result, data); + } + + #[test] + fn test_encryption_metadata_roundtrip() { + let meta = EncryptionMetadata { + algorithm: "AES256".to_string(), + nonce: "dGVzdG5vbmNlMTI=".to_string(), + encrypted_data_key: Some("c29tZWtleQ==".to_string()), + kms_key_id: None, + }; + let map = meta.to_metadata_map(); + let restored = EncryptionMetadata::from_metadata(&map).unwrap(); + assert_eq!(restored.algorithm, "AES256"); + assert_eq!(restored.nonce, meta.nonce); + assert_eq!(restored.encrypted_data_key, meta.encrypted_data_key); + } + + #[test] + fn test_is_encrypted() { + let mut meta = HashMap::new(); + assert!(!EncryptionMetadata::is_encrypted(&meta)); + meta.insert("x-amz-server-side-encryption".to_string(), "AES256".to_string()); + assert!(EncryptionMetadata::is_encrypted(&meta)); + } +} diff --git a/myfsio-engine/crates/myfsio-crypto/src/hashing.rs b/myfsio-engine/crates/myfsio-crypto/src/hashing.rs new file mode 100644 index 0000000..88d4f54 --- /dev/null +++ b/myfsio-engine/crates/myfsio-crypto/src/hashing.rs @@ -0,0 +1,132 @@ +use md5::{Digest, Md5}; +use sha2::Sha256; +use std::io::Read; +use std::path::Path; + +const CHUNK_SIZE: usize = 65536; + +pub fn md5_file(path: &Path) -> std::io::Result { + let mut file = std::fs::File::open(path)?; + let mut hasher = Md5::new(); + let mut buf = vec![0u8; CHUNK_SIZE]; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +pub fn md5_bytes(data: &[u8]) -> String { + let mut hasher = Md5::new(); + hasher.update(data); + format!("{:x}", hasher.finalize()) +} + +pub fn sha256_file(path: &Path) -> std::io::Result { + let mut file = std::fs::File::open(path)?; + let mut hasher = Sha256::new(); + let mut buf = vec![0u8; CHUNK_SIZE]; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +pub fn sha256_bytes(data: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(data); + format!("{:x}", hasher.finalize()) +} + +pub fn md5_sha256_file(path: &Path) -> std::io::Result<(String, String)> { + let mut file = std::fs::File::open(path)?; + let mut md5_hasher = Md5::new(); + let mut sha_hasher = Sha256::new(); + let mut buf = vec![0u8; CHUNK_SIZE]; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + md5_hasher.update(&buf[..n]); + sha_hasher.update(&buf[..n]); + } + Ok(( + format!("{:x}", md5_hasher.finalize()), + format!("{:x}", sha_hasher.finalize()), + )) +} + +pub async fn md5_file_async(path: &Path) -> std::io::Result { + let path = path.to_owned(); + tokio::task::spawn_blocking(move || md5_file(&path)) + .await + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))? +} + +pub async fn sha256_file_async(path: &Path) -> std::io::Result { + let path = path.to_owned(); + tokio::task::spawn_blocking(move || sha256_file(&path)) + .await + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))? +} + +pub async fn md5_sha256_file_async(path: &Path) -> std::io::Result<(String, String)> { + let path = path.to_owned(); + tokio::task::spawn_blocking(move || md5_sha256_file(&path)) + .await + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))? +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + #[test] + fn test_md5_bytes() { + assert_eq!(md5_bytes(b""), "d41d8cd98f00b204e9800998ecf8427e"); + assert_eq!(md5_bytes(b"hello"), "5d41402abc4b2a76b9719d911017c592"); + } + + #[test] + fn test_sha256_bytes() { + let hash = sha256_bytes(b"hello"); + assert_eq!(hash, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"); + } + + #[test] + fn test_md5_file() { + let mut tmp = tempfile::NamedTempFile::new().unwrap(); + tmp.write_all(b"hello").unwrap(); + tmp.flush().unwrap(); + let hash = md5_file(tmp.path()).unwrap(); + assert_eq!(hash, "5d41402abc4b2a76b9719d911017c592"); + } + + #[test] + fn test_md5_sha256_file() { + let mut tmp = tempfile::NamedTempFile::new().unwrap(); + tmp.write_all(b"hello").unwrap(); + tmp.flush().unwrap(); + let (md5, sha) = md5_sha256_file(tmp.path()).unwrap(); + assert_eq!(md5, "5d41402abc4b2a76b9719d911017c592"); + assert_eq!(sha, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"); + } + + #[tokio::test] + async fn test_md5_file_async() { + let mut tmp = tempfile::NamedTempFile::new().unwrap(); + tmp.write_all(b"hello").unwrap(); + tmp.flush().unwrap(); + let hash = md5_file_async(tmp.path()).await.unwrap(); + assert_eq!(hash, "5d41402abc4b2a76b9719d911017c592"); + } +} diff --git a/myfsio-engine/crates/myfsio-crypto/src/kms.rs b/myfsio-engine/crates/myfsio-crypto/src/kms.rs new file mode 100644 index 0000000..790c9fe --- /dev/null +++ b/myfsio-engine/crates/myfsio-crypto/src/kms.rs @@ -0,0 +1,453 @@ +use aes_gcm::aead::Aead; +use aes_gcm::{Aes256Gcm, KeyInit, Nonce}; +use base64::engine::general_purpose::STANDARD as B64; +use base64::Engine; +use chrono::{DateTime, Utc}; +use rand::RngCore; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use tokio::sync::RwLock; + +use crate::aes_gcm::CryptoError; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KmsKey { + #[serde(rename = "KeyId")] + pub key_id: String, + #[serde(rename = "Arn")] + pub arn: String, + #[serde(rename = "Description")] + pub description: String, + #[serde(rename = "CreationDate")] + pub creation_date: DateTime, + #[serde(rename = "Enabled")] + pub enabled: bool, + #[serde(rename = "KeyState")] + pub key_state: String, + #[serde(rename = "KeyUsage")] + pub key_usage: String, + #[serde(rename = "KeySpec")] + pub key_spec: String, + #[serde(rename = "EncryptedKeyMaterial")] + pub encrypted_key_material: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct KmsStore { + keys: Vec, +} + +pub struct KmsService { + keys_path: PathBuf, + master_key: Arc>, + keys: Arc>>, +} + +impl KmsService { + pub async fn new(keys_dir: &Path) -> Result { + std::fs::create_dir_all(keys_dir).map_err(CryptoError::Io)?; + + let keys_path = keys_dir.join("kms_keys.json"); + + let master_key = Self::load_or_create_master_key(&keys_dir.join("kms_master.key"))?; + + let keys = if keys_path.exists() { + let data = std::fs::read_to_string(&keys_path).map_err(CryptoError::Io)?; + let store: KmsStore = serde_json::from_str(&data) + .map_err(|e| CryptoError::EncryptionFailed(format!("Bad KMS store: {}", e)))?; + store.keys + } else { + Vec::new() + }; + + Ok(Self { + keys_path, + master_key: Arc::new(RwLock::new(master_key)), + keys: Arc::new(RwLock::new(keys)), + }) + } + + fn load_or_create_master_key(path: &Path) -> Result<[u8; 32], CryptoError> { + if path.exists() { + let encoded = std::fs::read_to_string(path).map_err(CryptoError::Io)?; + let decoded = B64.decode(encoded.trim()).map_err(|e| { + CryptoError::EncryptionFailed(format!("Bad master key encoding: {}", e)) + })?; + if decoded.len() != 32 { + return Err(CryptoError::InvalidKeySize(decoded.len())); + } + let mut key = [0u8; 32]; + key.copy_from_slice(&decoded); + Ok(key) + } else { + let mut key = [0u8; 32]; + rand::thread_rng().fill_bytes(&mut key); + let encoded = B64.encode(key); + std::fs::write(path, &encoded).map_err(CryptoError::Io)?; + Ok(key) + } + } + + fn encrypt_key_material( + master_key: &[u8; 32], + plaintext_key: &[u8], + ) -> Result { + let cipher = Aes256Gcm::new(master_key.into()); + let mut nonce_bytes = [0u8; 12]; + rand::thread_rng().fill_bytes(&mut nonce_bytes); + let nonce = Nonce::from_slice(&nonce_bytes); + + let ciphertext = cipher + .encrypt(nonce, plaintext_key) + .map_err(|e| CryptoError::EncryptionFailed(e.to_string()))?; + + let mut combined = Vec::with_capacity(12 + ciphertext.len()); + combined.extend_from_slice(&nonce_bytes); + combined.extend_from_slice(&ciphertext); + Ok(B64.encode(&combined)) + } + + fn decrypt_key_material( + master_key: &[u8; 32], + encrypted_b64: &str, + ) -> Result, CryptoError> { + let combined = B64.decode(encrypted_b64).map_err(|e| { + CryptoError::EncryptionFailed(format!("Bad key material encoding: {}", e)) + })?; + if combined.len() < 12 { + return Err(CryptoError::EncryptionFailed( + "Encrypted key material too short".to_string(), + )); + } + + let (nonce_bytes, ciphertext) = combined.split_at(12); + let cipher = Aes256Gcm::new(master_key.into()); + let nonce = Nonce::from_slice(nonce_bytes); + + cipher + .decrypt(nonce, ciphertext) + .map_err(|_| CryptoError::DecryptionFailed(0)) + } + + async fn save(&self) -> Result<(), CryptoError> { + let keys = self.keys.read().await; + let store = KmsStore { + keys: keys.clone(), + }; + let json = serde_json::to_string_pretty(&store) + .map_err(|e| CryptoError::EncryptionFailed(e.to_string()))?; + std::fs::write(&self.keys_path, json).map_err(CryptoError::Io)?; + Ok(()) + } + + pub async fn create_key(&self, description: &str) -> Result { + let key_id = uuid::Uuid::new_v4().to_string(); + let arn = format!("arn:aws:kms:local:000000000000:key/{}", key_id); + + let mut plaintext_key = [0u8; 32]; + rand::thread_rng().fill_bytes(&mut plaintext_key); + + let master = self.master_key.read().await; + let encrypted = Self::encrypt_key_material(&master, &plaintext_key)?; + + let kms_key = KmsKey { + key_id: key_id.clone(), + arn, + description: description.to_string(), + creation_date: Utc::now(), + enabled: true, + key_state: "Enabled".to_string(), + key_usage: "ENCRYPT_DECRYPT".to_string(), + key_spec: "SYMMETRIC_DEFAULT".to_string(), + encrypted_key_material: encrypted, + }; + + self.keys.write().await.push(kms_key.clone()); + self.save().await?; + Ok(kms_key) + } + + pub async fn list_keys(&self) -> Vec { + self.keys.read().await.clone() + } + + pub async fn get_key(&self, key_id: &str) -> Option { + let keys = self.keys.read().await; + keys.iter() + .find(|k| k.key_id == key_id || k.arn == key_id) + .cloned() + } + + pub async fn delete_key(&self, key_id: &str) -> Result { + let mut keys = self.keys.write().await; + let len_before = keys.len(); + keys.retain(|k| k.key_id != key_id && k.arn != key_id); + let removed = keys.len() < len_before; + drop(keys); + if removed { + self.save().await?; + } + Ok(removed) + } + + pub async fn enable_key(&self, key_id: &str) -> Result { + let mut keys = self.keys.write().await; + if let Some(key) = keys.iter_mut().find(|k| k.key_id == key_id) { + key.enabled = true; + key.key_state = "Enabled".to_string(); + drop(keys); + self.save().await?; + Ok(true) + } else { + Ok(false) + } + } + + pub async fn disable_key(&self, key_id: &str) -> Result { + let mut keys = self.keys.write().await; + if let Some(key) = keys.iter_mut().find(|k| k.key_id == key_id) { + key.enabled = false; + key.key_state = "Disabled".to_string(); + drop(keys); + self.save().await?; + Ok(true) + } else { + Ok(false) + } + } + + pub async fn decrypt_data_key(&self, key_id: &str) -> Result, CryptoError> { + let keys = self.keys.read().await; + let key = keys + .iter() + .find(|k| k.key_id == key_id || k.arn == key_id) + .ok_or_else(|| CryptoError::EncryptionFailed("KMS key not found".to_string()))?; + + if !key.enabled { + return Err(CryptoError::EncryptionFailed( + "KMS key is disabled".to_string(), + )); + } + + let master = self.master_key.read().await; + Self::decrypt_key_material(&master, &key.encrypted_key_material) + } + + pub async fn encrypt_data( + &self, + key_id: &str, + plaintext: &[u8], + ) -> Result, CryptoError> { + let data_key = self.decrypt_data_key(key_id).await?; + if data_key.len() != 32 { + return Err(CryptoError::InvalidKeySize(data_key.len())); + } + + let key_arr: [u8; 32] = data_key.try_into().unwrap(); + let cipher = Aes256Gcm::new(&key_arr.into()); + let mut nonce_bytes = [0u8; 12]; + rand::thread_rng().fill_bytes(&mut nonce_bytes); + let nonce = Nonce::from_slice(&nonce_bytes); + + let ciphertext = cipher + .encrypt(nonce, plaintext) + .map_err(|e| CryptoError::EncryptionFailed(e.to_string()))?; + + let mut result = Vec::with_capacity(12 + ciphertext.len()); + result.extend_from_slice(&nonce_bytes); + result.extend_from_slice(&ciphertext); + Ok(result) + } + + pub async fn decrypt_data( + &self, + key_id: &str, + ciphertext: &[u8], + ) -> Result, CryptoError> { + if ciphertext.len() < 12 { + return Err(CryptoError::EncryptionFailed( + "Ciphertext too short".to_string(), + )); + } + + let data_key = self.decrypt_data_key(key_id).await?; + if data_key.len() != 32 { + return Err(CryptoError::InvalidKeySize(data_key.len())); + } + + let key_arr: [u8; 32] = data_key.try_into().unwrap(); + let (nonce_bytes, ct) = ciphertext.split_at(12); + let cipher = Aes256Gcm::new(&key_arr.into()); + let nonce = Nonce::from_slice(nonce_bytes); + + cipher + .decrypt(nonce, ct) + .map_err(|_| CryptoError::DecryptionFailed(0)) + } + + pub async fn generate_data_key( + &self, + key_id: &str, + num_bytes: usize, + ) -> Result<(Vec, Vec), CryptoError> { + let kms_key = self.decrypt_data_key(key_id).await?; + if kms_key.len() != 32 { + return Err(CryptoError::InvalidKeySize(kms_key.len())); + } + + let mut plaintext_key = vec![0u8; num_bytes]; + rand::thread_rng().fill_bytes(&mut plaintext_key); + + let key_arr: [u8; 32] = kms_key.try_into().unwrap(); + let cipher = Aes256Gcm::new(&key_arr.into()); + let mut nonce_bytes = [0u8; 12]; + rand::thread_rng().fill_bytes(&mut nonce_bytes); + let nonce = Nonce::from_slice(&nonce_bytes); + + let encrypted = cipher + .encrypt(nonce, plaintext_key.as_slice()) + .map_err(|e| CryptoError::EncryptionFailed(e.to_string()))?; + + let mut wrapped = Vec::with_capacity(12 + encrypted.len()); + wrapped.extend_from_slice(&nonce_bytes); + wrapped.extend_from_slice(&encrypted); + + Ok((plaintext_key, wrapped)) + } +} + +pub async fn load_or_create_master_key(keys_dir: &Path) -> Result<[u8; 32], CryptoError> { + std::fs::create_dir_all(keys_dir).map_err(CryptoError::Io)?; + let path = keys_dir.join("master.key"); + + if path.exists() { + let encoded = std::fs::read_to_string(&path).map_err(CryptoError::Io)?; + let decoded = B64.decode(encoded.trim()).map_err(|e| { + CryptoError::EncryptionFailed(format!("Bad master key encoding: {}", e)) + })?; + if decoded.len() != 32 { + return Err(CryptoError::InvalidKeySize(decoded.len())); + } + let mut key = [0u8; 32]; + key.copy_from_slice(&decoded); + Ok(key) + } else { + let mut key = [0u8; 32]; + rand::thread_rng().fill_bytes(&mut key); + let encoded = B64.encode(key); + std::fs::write(&path, &encoded).map_err(CryptoError::Io)?; + Ok(key) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_create_and_list_keys() { + let dir = tempfile::tempdir().unwrap(); + let kms = KmsService::new(dir.path()).await.unwrap(); + + let key = kms.create_key("test key").await.unwrap(); + assert!(key.enabled); + assert_eq!(key.description, "test key"); + assert!(key.key_id.len() > 0); + + let keys = kms.list_keys().await; + assert_eq!(keys.len(), 1); + assert_eq!(keys[0].key_id, key.key_id); + } + + #[tokio::test] + async fn test_enable_disable_key() { + let dir = tempfile::tempdir().unwrap(); + let kms = KmsService::new(dir.path()).await.unwrap(); + + let key = kms.create_key("toggle").await.unwrap(); + assert!(key.enabled); + + kms.disable_key(&key.key_id).await.unwrap(); + let k = kms.get_key(&key.key_id).await.unwrap(); + assert!(!k.enabled); + + kms.enable_key(&key.key_id).await.unwrap(); + let k = kms.get_key(&key.key_id).await.unwrap(); + assert!(k.enabled); + } + + #[tokio::test] + async fn test_delete_key() { + let dir = tempfile::tempdir().unwrap(); + let kms = KmsService::new(dir.path()).await.unwrap(); + + let key = kms.create_key("doomed").await.unwrap(); + assert!(kms.delete_key(&key.key_id).await.unwrap()); + assert!(kms.get_key(&key.key_id).await.is_none()); + assert_eq!(kms.list_keys().await.len(), 0); + } + + #[tokio::test] + async fn test_encrypt_decrypt_data() { + let dir = tempfile::tempdir().unwrap(); + let kms = KmsService::new(dir.path()).await.unwrap(); + + let key = kms.create_key("enc-key").await.unwrap(); + let plaintext = b"Hello, KMS!"; + + let ciphertext = kms.encrypt_data(&key.key_id, plaintext).await.unwrap(); + assert_ne!(&ciphertext, plaintext); + + let decrypted = kms.decrypt_data(&key.key_id, &ciphertext).await.unwrap(); + assert_eq!(decrypted, plaintext); + } + + #[tokio::test] + async fn test_generate_data_key() { + let dir = tempfile::tempdir().unwrap(); + let kms = KmsService::new(dir.path()).await.unwrap(); + + let key = kms.create_key("data-key-gen").await.unwrap(); + let (plaintext, wrapped) = kms.generate_data_key(&key.key_id, 32).await.unwrap(); + + assert_eq!(plaintext.len(), 32); + assert!(wrapped.len() > 32); + } + + #[tokio::test] + async fn test_disabled_key_cannot_encrypt() { + let dir = tempfile::tempdir().unwrap(); + let kms = KmsService::new(dir.path()).await.unwrap(); + + let key = kms.create_key("disabled").await.unwrap(); + kms.disable_key(&key.key_id).await.unwrap(); + + let result = kms.encrypt_data(&key.key_id, b"test").await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_persistence_across_reload() { + let dir = tempfile::tempdir().unwrap(); + + let key_id = { + let kms = KmsService::new(dir.path()).await.unwrap(); + let key = kms.create_key("persistent").await.unwrap(); + key.key_id + }; + + let kms2 = KmsService::new(dir.path()).await.unwrap(); + let key = kms2.get_key(&key_id).await; + assert!(key.is_some()); + assert_eq!(key.unwrap().description, "persistent"); + } + + #[tokio::test] + async fn test_master_key_roundtrip() { + let dir = tempfile::tempdir().unwrap(); + let key1 = load_or_create_master_key(dir.path()).await.unwrap(); + let key2 = load_or_create_master_key(dir.path()).await.unwrap(); + assert_eq!(key1, key2); + } +} diff --git a/myfsio-engine/crates/myfsio-crypto/src/lib.rs b/myfsio-engine/crates/myfsio-crypto/src/lib.rs new file mode 100644 index 0000000..402bb7a --- /dev/null +++ b/myfsio-engine/crates/myfsio-crypto/src/lib.rs @@ -0,0 +1,4 @@ +pub mod hashing; +pub mod aes_gcm; +pub mod kms; +pub mod encryption; diff --git a/myfsio-engine/crates/myfsio-server/Cargo.toml b/myfsio-engine/crates/myfsio-server/Cargo.toml new file mode 100644 index 0000000..7f1f9df --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "myfsio-server" +version = "0.1.0" +edition = "2021" + +[dependencies] +myfsio-common = { path = "../myfsio-common" } +myfsio-auth = { path = "../myfsio-auth" } +myfsio-crypto = { path = "../myfsio-crypto" } +myfsio-storage = { path = "../myfsio-storage" } +myfsio-xml = { path = "../myfsio-xml" } +base64 = { workspace = true } +axum = { workspace = true } +tokio = { workspace = true } +tower = { workspace = true } +tower-http = { workspace = true } +hyper = { workspace = true } +bytes = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +tokio-util = { workspace = true } +chrono = { workspace = true } +uuid = { workspace = true } +futures = { workspace = true } +http-body-util = "0.1" +percent-encoding = { workspace = true } +quick-xml = { workspace = true } + +[dev-dependencies] +tempfile = "3" +tower = { workspace = true, features = ["util"] } diff --git a/myfsio-engine/crates/myfsio-server/src/config.rs b/myfsio-engine/crates/myfsio-server/src/config.rs new file mode 100644 index 0000000..fea4d56 --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/src/config.rs @@ -0,0 +1,111 @@ +use std::net::SocketAddr; +use std::path::PathBuf; + +#[derive(Debug, Clone)] +pub struct ServerConfig { + pub bind_addr: SocketAddr, + pub storage_root: PathBuf, + pub region: String, + pub iam_config_path: PathBuf, + pub sigv4_timestamp_tolerance_secs: u64, + pub presigned_url_min_expiry: u64, + pub presigned_url_max_expiry: u64, + pub secret_key: Option, + pub encryption_enabled: bool, + pub kms_enabled: bool, + pub gc_enabled: bool, + pub integrity_enabled: bool, + pub metrics_enabled: bool, + pub lifecycle_enabled: bool, +} + +impl ServerConfig { + pub fn from_env() -> Self { + let host = std::env::var("HOST").unwrap_or_else(|_| "127.0.0.1".to_string()); + let port: u16 = std::env::var("PORT") + .unwrap_or_else(|_| "5000".to_string()) + .parse() + .unwrap_or(5000); + let storage_root = std::env::var("STORAGE_ROOT") + .unwrap_or_else(|_| "./data".to_string()); + let region = std::env::var("AWS_REGION") + .unwrap_or_else(|_| "us-east-1".to_string()); + + let storage_path = PathBuf::from(&storage_root); + let iam_config_path = std::env::var("IAM_CONFIG") + .map(PathBuf::from) + .unwrap_or_else(|_| { + storage_path.join(".myfsio.sys").join("config").join("iam.json") + }); + + let sigv4_timestamp_tolerance_secs: u64 = std::env::var("SIGV4_TIMESTAMP_TOLERANCE_SECONDS") + .unwrap_or_else(|_| "900".to_string()) + .parse() + .unwrap_or(900); + + let presigned_url_min_expiry: u64 = std::env::var("PRESIGNED_URL_MIN_EXPIRY_SECONDS") + .unwrap_or_else(|_| "1".to_string()) + .parse() + .unwrap_or(1); + + let presigned_url_max_expiry: u64 = std::env::var("PRESIGNED_URL_MAX_EXPIRY_SECONDS") + .unwrap_or_else(|_| "604800".to_string()) + .parse() + .unwrap_or(604800); + + let secret_key = { + let env_key = std::env::var("SECRET_KEY").ok(); + match env_key { + Some(k) if !k.is_empty() && k != "dev-secret-key" => Some(k), + _ => { + let secret_file = storage_path + .join(".myfsio.sys") + .join("config") + .join(".secret"); + std::fs::read_to_string(&secret_file).ok().map(|s| s.trim().to_string()) + } + } + }; + + let encryption_enabled = std::env::var("ENCRYPTION_ENABLED") + .unwrap_or_else(|_| "false".to_string()) + .to_lowercase() == "true"; + + let kms_enabled = std::env::var("KMS_ENABLED") + .unwrap_or_else(|_| "false".to_string()) + .to_lowercase() == "true"; + + let gc_enabled = std::env::var("GC_ENABLED") + .unwrap_or_else(|_| "false".to_string()) + .to_lowercase() == "true"; + + let integrity_enabled = std::env::var("INTEGRITY_ENABLED") + .unwrap_or_else(|_| "false".to_string()) + .to_lowercase() == "true"; + + let metrics_enabled = std::env::var("OPERATION_METRICS_ENABLED") + .unwrap_or_else(|_| "false".to_string()) + .to_lowercase() == "true"; + + let lifecycle_enabled = std::env::var("LIFECYCLE_ENABLED") + .unwrap_or_else(|_| "false".to_string()) + .to_lowercase() == "true"; + + Self { + bind_addr: SocketAddr::new(host.parse().unwrap(), port), + storage_root: storage_path, + region, + iam_config_path, + sigv4_timestamp_tolerance_secs, + presigned_url_min_expiry, + presigned_url_max_expiry, + secret_key, + encryption_enabled, + kms_enabled, + gc_enabled, + integrity_enabled, + metrics_enabled, + lifecycle_enabled, + } + } +} diff --git a/myfsio-engine/crates/myfsio-server/src/handlers/config.rs b/myfsio-engine/crates/myfsio-server/src/handlers/config.rs new file mode 100644 index 0000000..01dac09 --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/src/handlers/config.rs @@ -0,0 +1,623 @@ +use axum::body::Body; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; + +use myfsio_common::error::{S3Error, S3ErrorCode}; +use myfsio_storage::traits::StorageEngine; + +use crate::state::AppState; + +fn xml_response(status: StatusCode, xml: String) -> Response { + (status, [("content-type", "application/xml")], xml).into_response() +} + +fn storage_err(err: myfsio_storage::error::StorageError) -> Response { + let s3err = S3Error::from(err); + let status = StatusCode::from_u16(s3err.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + (status, [("content-type", "application/xml")], s3err.to_xml()).into_response() +} + +pub async fn get_versioning(state: &AppState, bucket: &str) -> Response { + match state.storage.is_versioning_enabled(bucket).await { + Ok(enabled) => { + let status_str = if enabled { "Enabled" } else { "Suspended" }; + let xml = format!( + "\ + \ + {}\ + ", + status_str + ); + xml_response(StatusCode::OK, xml) + } + Err(e) => storage_err(e), + } +} + +pub async fn put_versioning(state: &AppState, bucket: &str, body: Body) -> Response { + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(collected) => collected.to_bytes(), + Err(_) => { + return xml_response( + StatusCode::BAD_REQUEST, + S3Error::from_code(S3ErrorCode::MalformedXML).to_xml(), + ); + } + }; + + let xml_str = String::from_utf8_lossy(&body_bytes); + let enabled = xml_str.contains("Enabled"); + + match state.storage.set_versioning(bucket, enabled).await { + Ok(()) => StatusCode::OK.into_response(), + Err(e) => storage_err(e), + } +} + +pub async fn get_tagging(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(config) => { + let mut xml = String::from( + "\ + " + ); + for tag in &config.tags { + xml.push_str(&format!( + "{}{}", + tag.key, tag.value + )); + } + xml.push_str(""); + xml_response(StatusCode::OK, xml) + } + Err(e) => storage_err(e), + } +} + +pub async fn put_tagging(state: &AppState, bucket: &str, body: Body) -> Response { + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(collected) => collected.to_bytes(), + Err(_) => { + return xml_response( + StatusCode::BAD_REQUEST, + S3Error::from_code(S3ErrorCode::MalformedXML).to_xml(), + ); + } + }; + + let xml_str = String::from_utf8_lossy(&body_bytes); + let tags = parse_tagging_xml(&xml_str); + + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.tags = tags; + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::OK.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn delete_tagging(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.tags.clear(); + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn get_cors(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(config) => { + if let Some(cors) = &config.cors { + xml_response(StatusCode::OK, cors.to_string()) + } else { + xml_response( + StatusCode::NOT_FOUND, + S3Error::new(S3ErrorCode::NoSuchKey, "The CORS configuration does not exist").to_xml(), + ) + } + } + Err(e) => storage_err(e), + } +} + +pub async fn put_cors(state: &AppState, bucket: &str, body: Body) -> Response { + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(collected) => collected.to_bytes(), + Err(_) => return StatusCode::BAD_REQUEST.into_response(), + }; + + let body_str = String::from_utf8_lossy(&body_bytes); + let value = serde_json::Value::String(body_str.to_string()); + + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.cors = Some(value); + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::OK.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn delete_cors(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.cors = None; + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn get_location(state: &AppState, _bucket: &str) -> Response { + let xml = format!( + "\ + {}", + state.config.region + ); + xml_response(StatusCode::OK, xml) +} + +pub async fn get_encryption(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(config) => { + if let Some(enc) = &config.encryption { + xml_response(StatusCode::OK, enc.to_string()) + } else { + xml_response( + StatusCode::NOT_FOUND, + S3Error::new( + S3ErrorCode::InvalidRequest, + "The server side encryption configuration was not found", + ).to_xml(), + ) + } + } + Err(e) => storage_err(e), + } +} + +pub async fn put_encryption(state: &AppState, bucket: &str, body: Body) -> Response { + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(collected) => collected.to_bytes(), + Err(_) => return StatusCode::BAD_REQUEST.into_response(), + }; + let value = serde_json::Value::String(String::from_utf8_lossy(&body_bytes).to_string()); + + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.encryption = Some(value); + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::OK.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn delete_encryption(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.encryption = None; + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn get_lifecycle(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(config) => { + if let Some(lc) = &config.lifecycle { + xml_response(StatusCode::OK, lc.to_string()) + } else { + xml_response( + StatusCode::NOT_FOUND, + S3Error::new(S3ErrorCode::NoSuchKey, "The lifecycle configuration does not exist").to_xml(), + ) + } + } + Err(e) => storage_err(e), + } +} + +pub async fn put_lifecycle(state: &AppState, bucket: &str, body: Body) -> Response { + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(collected) => collected.to_bytes(), + Err(_) => return StatusCode::BAD_REQUEST.into_response(), + }; + let value = serde_json::Value::String(String::from_utf8_lossy(&body_bytes).to_string()); + + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.lifecycle = Some(value); + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::OK.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn delete_lifecycle(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.lifecycle = None; + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn get_acl(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(config) => { + if let Some(acl) = &config.acl { + xml_response(StatusCode::OK, acl.to_string()) + } else { + let xml = "\ + \ + myfsiomyfsio\ + \ + \ + myfsiomyfsio\ + FULL_CONTROL\ + "; + xml_response(StatusCode::OK, xml.to_string()) + } + } + Err(e) => storage_err(e), + } +} + +pub async fn put_acl(state: &AppState, bucket: &str, body: Body) -> Response { + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(collected) => collected.to_bytes(), + Err(_) => return StatusCode::BAD_REQUEST.into_response(), + }; + let value = serde_json::Value::String(String::from_utf8_lossy(&body_bytes).to_string()); + + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.acl = Some(value); + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::OK.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn get_website(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(config) => { + if let Some(ws) = &config.website { + xml_response(StatusCode::OK, ws.to_string()) + } else { + xml_response( + StatusCode::NOT_FOUND, + S3Error::new(S3ErrorCode::NoSuchKey, "The website configuration does not exist").to_xml(), + ) + } + } + Err(e) => storage_err(e), + } +} + +pub async fn put_website(state: &AppState, bucket: &str, body: Body) -> Response { + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(collected) => collected.to_bytes(), + Err(_) => return StatusCode::BAD_REQUEST.into_response(), + }; + let value = serde_json::Value::String(String::from_utf8_lossy(&body_bytes).to_string()); + + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.website = Some(value); + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::OK.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn delete_website(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(mut config) => { + config.website = None; + match state.storage.set_bucket_config(bucket, &config).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => storage_err(e), + } + } + Err(e) => storage_err(e), + } +} + +pub async fn get_object_lock(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(config) => { + if let Some(ol) = &config.object_lock { + xml_response(StatusCode::OK, ol.to_string()) + } else { + let xml = "\ + \ + Disabled\ + "; + xml_response(StatusCode::OK, xml.to_string()) + } + } + Err(e) => storage_err(e), + } +} + +pub async fn get_notification(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(config) => { + if let Some(n) = &config.notification { + xml_response(StatusCode::OK, n.to_string()) + } else { + let xml = "\ + \ + "; + xml_response(StatusCode::OK, xml.to_string()) + } + } + Err(e) => storage_err(e), + } +} + +pub async fn get_logging(state: &AppState, bucket: &str) -> Response { + match state.storage.get_bucket_config(bucket).await { + Ok(config) => { + if let Some(l) = &config.logging { + xml_response(StatusCode::OK, l.to_string()) + } else { + let xml = "\ + \ + "; + xml_response(StatusCode::OK, xml.to_string()) + } + } + Err(e) => storage_err(e), + } +} + +pub async fn list_object_versions(state: &AppState, bucket: &str) -> Response { + match state.storage.list_buckets().await { + Ok(buckets) => { + if !buckets.iter().any(|b| b.name == bucket) { + return storage_err(myfsio_storage::error::StorageError::BucketNotFound( + bucket.to_string(), + )); + } + } + Err(e) => return storage_err(e), + } + + let params = myfsio_common::types::ListParams { + max_keys: 1000, + ..Default::default() + }; + + let objects = match state.storage.list_objects(bucket, ¶ms).await { + Ok(result) => result.objects, + Err(e) => return storage_err(e), + }; + + let mut xml = String::from( + "\ + " + ); + xml.push_str(&format!("{}", bucket)); + + for obj in &objects { + xml.push_str(""); + xml.push_str(&format!("{}", obj.key)); + xml.push_str("null"); + xml.push_str("true"); + xml.push_str(&format!( + "{}", + obj.last_modified.to_rfc3339() + )); + if let Some(ref etag) = obj.etag { + xml.push_str(&format!("\"{}\"", etag)); + } + xml.push_str(&format!("{}", obj.size)); + xml.push_str("STANDARD"); + xml.push_str(""); + } + + xml.push_str(""); + xml_response(StatusCode::OK, xml) +} + +pub async fn get_object_tagging(state: &AppState, bucket: &str, key: &str) -> Response { + match state.storage.get_object_tags(bucket, key).await { + Ok(tags) => { + let mut xml = String::from( + "\ + " + ); + for tag in &tags { + xml.push_str(&format!( + "{}{}", + tag.key, tag.value + )); + } + xml.push_str(""); + xml_response(StatusCode::OK, xml) + } + Err(e) => storage_err(e), + } +} + +pub async fn put_object_tagging(state: &AppState, bucket: &str, key: &str, body: Body) -> Response { + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(collected) => collected.to_bytes(), + Err(_) => { + return xml_response( + StatusCode::BAD_REQUEST, + S3Error::from_code(S3ErrorCode::MalformedXML).to_xml(), + ); + } + }; + + let xml_str = String::from_utf8_lossy(&body_bytes); + let tags = parse_tagging_xml(&xml_str); + + match state.storage.set_object_tags(bucket, key, &tags).await { + Ok(()) => StatusCode::OK.into_response(), + Err(e) => storage_err(e), + } +} + +pub async fn delete_object_tagging(state: &AppState, bucket: &str, key: &str) -> Response { + match state.storage.delete_object_tags(bucket, key).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => storage_err(e), + } +} + +pub async fn get_object_acl(state: &AppState, bucket: &str, key: &str) -> Response { + match state.storage.head_object(bucket, key).await { + Ok(_) => { + let xml = "\ + \ + myfsiomyfsio\ + \ + \ + myfsiomyfsio\ + FULL_CONTROL\ + "; + xml_response(StatusCode::OK, xml.to_string()) + } + Err(e) => storage_err(e), + } +} + +pub async fn put_object_acl(state: &AppState, bucket: &str, key: &str, _body: Body) -> Response { + match state.storage.head_object(bucket, key).await { + Ok(_) => StatusCode::OK.into_response(), + Err(e) => storage_err(e), + } +} + +pub async fn get_object_retention(state: &AppState, bucket: &str, key: &str) -> Response { + match state.storage.head_object(bucket, key).await { + Ok(_) => { + xml_response( + StatusCode::NOT_FOUND, + S3Error::new( + S3ErrorCode::InvalidRequest, + "No retention policy configured", + ).to_xml(), + ) + } + Err(e) => storage_err(e), + } +} + +pub async fn put_object_retention(state: &AppState, bucket: &str, key: &str, _body: Body) -> Response { + match state.storage.head_object(bucket, key).await { + Ok(_) => StatusCode::OK.into_response(), + Err(e) => storage_err(e), + } +} + +pub async fn get_object_legal_hold(state: &AppState, bucket: &str, key: &str) -> Response { + match state.storage.head_object(bucket, key).await { + Ok(_) => { + let xml = "\ + \ + OFF"; + xml_response(StatusCode::OK, xml.to_string()) + } + Err(e) => storage_err(e), + } +} + +pub async fn put_object_legal_hold(state: &AppState, bucket: &str, key: &str, _body: Body) -> Response { + match state.storage.head_object(bucket, key).await { + Ok(_) => StatusCode::OK.into_response(), + Err(e) => storage_err(e), + } +} + +fn parse_tagging_xml(xml: &str) -> Vec { + let mut tags = Vec::new(); + let mut in_tag = false; + let mut current_key = String::new(); + let mut current_value = String::new(); + let mut current_element = String::new(); + + let mut reader = quick_xml::Reader::from_str(xml); + let mut buf = Vec::new(); + + loop { + match reader.read_event_into(&mut buf) { + Ok(quick_xml::events::Event::Start(ref e)) => { + let name = String::from_utf8_lossy(e.name().as_ref()).to_string(); + current_element = name.clone(); + if name == "Tag" { + in_tag = true; + current_key.clear(); + current_value.clear(); + } + } + Ok(quick_xml::events::Event::Text(ref e)) => { + if in_tag { + let text = e.unescape().unwrap_or_default().to_string(); + match current_element.as_str() { + "Key" => current_key = text, + "Value" => current_value = text, + _ => {} + } + } + } + Ok(quick_xml::events::Event::End(ref e)) => { + let name = String::from_utf8_lossy(e.name().as_ref()).to_string(); + if name == "Tag" && in_tag { + if !current_key.is_empty() { + tags.push(myfsio_common::types::Tag { + key: current_key.clone(), + value: current_value.clone(), + }); + } + in_tag = false; + } + } + Ok(quick_xml::events::Event::Eof) => break, + Err(_) => break, + _ => {} + } + buf.clear(); + } + + tags +} diff --git a/myfsio-engine/crates/myfsio-server/src/handlers/kms.rs b/myfsio-engine/crates/myfsio-server/src/handlers/kms.rs new file mode 100644 index 0000000..0a4fba0 --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/src/handlers/kms.rs @@ -0,0 +1,278 @@ +use axum::body::Body; +use axum::extract::State; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; +use base64::engine::general_purpose::STANDARD as B64; +use base64::Engine; +use serde_json::json; + +use crate::state::AppState; + +fn json_ok(value: serde_json::Value) -> Response { + ( + StatusCode::OK, + [("content-type", "application/json")], + value.to_string(), + ) + .into_response() +} + +fn json_err(status: StatusCode, msg: &str) -> Response { + ( + status, + [("content-type", "application/json")], + json!({"error": msg}).to_string(), + ) + .into_response() +} + +pub async fn list_keys(State(state): State) -> Response { + let kms = match &state.kms { + Some(k) => k, + None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"), + }; + + let keys = kms.list_keys().await; + let keys_json: Vec = keys + .iter() + .map(|k| { + json!({ + "KeyId": k.key_id, + "Arn": k.arn, + "Description": k.description, + "CreationDate": k.creation_date.to_rfc3339(), + "Enabled": k.enabled, + "KeyState": k.key_state, + "KeyUsage": k.key_usage, + "KeySpec": k.key_spec, + }) + }) + .collect(); + + json_ok(json!({"keys": keys_json})) +} + +pub async fn create_key(State(state): State, body: Body) -> Response { + let kms = match &state.kms { + Some(k) => k, + None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"), + }; + + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(c) => c.to_bytes(), + Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid request body"), + }; + + let description = if body_bytes.is_empty() { + String::new() + } else { + match serde_json::from_slice::(&body_bytes) { + Ok(v) => v + .get("Description") + .or_else(|| v.get("description")) + .and_then(|d| d.as_str()) + .unwrap_or("") + .to_string(), + Err(_) => String::new(), + } + }; + + match kms.create_key(&description).await { + Ok(key) => json_ok(json!({ + "KeyId": key.key_id, + "Arn": key.arn, + "Description": key.description, + "CreationDate": key.creation_date.to_rfc3339(), + "Enabled": key.enabled, + "KeyState": key.key_state, + })), + Err(e) => json_err(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), + } +} + +pub async fn get_key( + State(state): State, + axum::extract::Path(key_id): axum::extract::Path, +) -> Response { + let kms = match &state.kms { + Some(k) => k, + None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"), + }; + + match kms.get_key(&key_id).await { + Some(key) => json_ok(json!({ + "KeyId": key.key_id, + "Arn": key.arn, + "Description": key.description, + "CreationDate": key.creation_date.to_rfc3339(), + "Enabled": key.enabled, + "KeyState": key.key_state, + "KeyUsage": key.key_usage, + "KeySpec": key.key_spec, + })), + None => json_err(StatusCode::NOT_FOUND, "Key not found"), + } +} + +pub async fn delete_key( + State(state): State, + axum::extract::Path(key_id): axum::extract::Path, +) -> Response { + let kms = match &state.kms { + Some(k) => k, + None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"), + }; + + match kms.delete_key(&key_id).await { + Ok(true) => StatusCode::NO_CONTENT.into_response(), + Ok(false) => json_err(StatusCode::NOT_FOUND, "Key not found"), + Err(e) => json_err(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), + } +} + +pub async fn enable_key( + State(state): State, + axum::extract::Path(key_id): axum::extract::Path, +) -> Response { + let kms = match &state.kms { + Some(k) => k, + None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"), + }; + + match kms.enable_key(&key_id).await { + Ok(true) => json_ok(json!({"status": "enabled"})), + Ok(false) => json_err(StatusCode::NOT_FOUND, "Key not found"), + Err(e) => json_err(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), + } +} + +pub async fn disable_key( + State(state): State, + axum::extract::Path(key_id): axum::extract::Path, +) -> Response { + let kms = match &state.kms { + Some(k) => k, + None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"), + }; + + match kms.disable_key(&key_id).await { + Ok(true) => json_ok(json!({"status": "disabled"})), + Ok(false) => json_err(StatusCode::NOT_FOUND, "Key not found"), + Err(e) => json_err(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), + } +} + +pub async fn encrypt(State(state): State, body: Body) -> Response { + let kms = match &state.kms { + Some(k) => k, + None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"), + }; + + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(c) => c.to_bytes(), + Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid request body"), + }; + + let req: serde_json::Value = match serde_json::from_slice(&body_bytes) { + Ok(v) => v, + Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid JSON"), + }; + + let key_id = match req.get("KeyId").and_then(|v| v.as_str()) { + Some(k) => k, + None => return json_err(StatusCode::BAD_REQUEST, "Missing KeyId"), + }; + let plaintext_b64 = match req.get("Plaintext").and_then(|v| v.as_str()) { + Some(p) => p, + None => return json_err(StatusCode::BAD_REQUEST, "Missing Plaintext"), + }; + let plaintext = match B64.decode(plaintext_b64) { + Ok(p) => p, + Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid base64 Plaintext"), + }; + + match kms.encrypt_data(key_id, &plaintext).await { + Ok(ct) => json_ok(json!({ + "KeyId": key_id, + "CiphertextBlob": B64.encode(&ct), + })), + Err(e) => json_err(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), + } +} + +pub async fn decrypt(State(state): State, body: Body) -> Response { + let kms = match &state.kms { + Some(k) => k, + None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"), + }; + + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(c) => c.to_bytes(), + Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid request body"), + }; + + let req: serde_json::Value = match serde_json::from_slice(&body_bytes) { + Ok(v) => v, + Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid JSON"), + }; + + let key_id = match req.get("KeyId").and_then(|v| v.as_str()) { + Some(k) => k, + None => return json_err(StatusCode::BAD_REQUEST, "Missing KeyId"), + }; + let ct_b64 = match req.get("CiphertextBlob").and_then(|v| v.as_str()) { + Some(c) => c, + None => return json_err(StatusCode::BAD_REQUEST, "Missing CiphertextBlob"), + }; + let ciphertext = match B64.decode(ct_b64) { + Ok(c) => c, + Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid base64"), + }; + + match kms.decrypt_data(key_id, &ciphertext).await { + Ok(pt) => json_ok(json!({ + "KeyId": key_id, + "Plaintext": B64.encode(&pt), + })), + Err(e) => json_err(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), + } +} + +pub async fn generate_data_key(State(state): State, body: Body) -> Response { + let kms = match &state.kms { + Some(k) => k, + None => return json_err(StatusCode::SERVICE_UNAVAILABLE, "KMS not enabled"), + }; + + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(c) => c.to_bytes(), + Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid request body"), + }; + + let req: serde_json::Value = match serde_json::from_slice(&body_bytes) { + Ok(v) => v, + Err(_) => return json_err(StatusCode::BAD_REQUEST, "Invalid JSON"), + }; + + let key_id = match req.get("KeyId").and_then(|v| v.as_str()) { + Some(k) => k, + None => return json_err(StatusCode::BAD_REQUEST, "Missing KeyId"), + }; + let num_bytes = req + .get("NumberOfBytes") + .and_then(|v| v.as_u64()) + .unwrap_or(32) as usize; + + if num_bytes < 1 || num_bytes > 1024 { + return json_err(StatusCode::BAD_REQUEST, "NumberOfBytes must be 1-1024"); + } + + match kms.generate_data_key(key_id, num_bytes).await { + Ok((plaintext, wrapped)) => json_ok(json!({ + "KeyId": key_id, + "Plaintext": B64.encode(&plaintext), + "CiphertextBlob": B64.encode(&wrapped), + })), + Err(e) => json_err(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), + } +} diff --git a/myfsio-engine/crates/myfsio-server/src/handlers/mod.rs b/myfsio-engine/crates/myfsio-server/src/handlers/mod.rs new file mode 100644 index 0000000..9055a0a --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/src/handlers/mod.rs @@ -0,0 +1,1027 @@ +mod config; +pub mod kms; + +use std::collections::HashMap; + +use axum::body::Body; +use axum::extract::{Path, Query, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::{IntoResponse, Response}; + +use myfsio_common::error::S3Error; +use myfsio_common::types::PartInfo; +use myfsio_storage::traits::StorageEngine; +use tokio::io::AsyncSeekExt; +use tokio_util::io::ReaderStream; + +use crate::state::AppState; + +fn s3_error_response(err: S3Error) -> Response { + let status = StatusCode::from_u16(err.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + let body = err.to_xml(); + ( + status, + [("content-type", "application/xml")], + body, + ) + .into_response() +} + +fn storage_err_response(err: myfsio_storage::error::StorageError) -> Response { + s3_error_response(S3Error::from(err)) +} + +pub async fn list_buckets(State(state): State) -> Response { + match state.storage.list_buckets().await { + Ok(buckets) => { + let xml = myfsio_xml::response::list_buckets_xml("myfsio", "myfsio", &buckets); + ( + StatusCode::OK, + [("content-type", "application/xml")], + xml, + ) + .into_response() + } + Err(e) => storage_err_response(e), + } +} + +pub async fn create_bucket( + State(state): State, + Path(bucket): Path, + Query(query): Query, + body: Body, +) -> Response { + if query.versioning.is_some() { + return config::put_versioning(&state, &bucket, body).await; + } + if query.tagging.is_some() { + return config::put_tagging(&state, &bucket, body).await; + } + if query.cors.is_some() { + return config::put_cors(&state, &bucket, body).await; + } + if query.encryption.is_some() { + return config::put_encryption(&state, &bucket, body).await; + } + if query.lifecycle.is_some() { + return config::put_lifecycle(&state, &bucket, body).await; + } + if query.acl.is_some() { + return config::put_acl(&state, &bucket, body).await; + } + if query.website.is_some() { + return config::put_website(&state, &bucket, body).await; + } + + match state.storage.create_bucket(&bucket).await { + Ok(()) => { + ( + StatusCode::OK, + [("location", format!("/{}", bucket).as_str())], + "", + ) + .into_response() + } + Err(e) => storage_err_response(e), + } +} + +#[derive(serde::Deserialize, Default)] +pub struct BucketQuery { + #[serde(rename = "list-type")] + pub list_type: Option, + pub prefix: Option, + pub delimiter: Option, + #[serde(rename = "max-keys")] + pub max_keys: Option, + #[serde(rename = "continuation-token")] + pub continuation_token: Option, + #[serde(rename = "start-after")] + pub start_after: Option, + pub uploads: Option, + pub delete: Option, + pub versioning: Option, + pub tagging: Option, + pub cors: Option, + pub location: Option, + pub encryption: Option, + pub lifecycle: Option, + pub acl: Option, + pub policy: Option, + pub website: Option, + #[serde(rename = "object-lock")] + pub object_lock: Option, + pub notification: Option, + pub logging: Option, + pub versions: Option, +} + +pub async fn get_bucket( + State(state): State, + Path(bucket): Path, + Query(query): Query, +) -> Response { + if !matches!(state.storage.bucket_exists(&bucket).await, Ok(true)) { + return storage_err_response( + myfsio_storage::error::StorageError::BucketNotFound(bucket), + ); + } + + if query.versioning.is_some() { + return config::get_versioning(&state, &bucket).await; + } + if query.tagging.is_some() { + return config::get_tagging(&state, &bucket).await; + } + if query.cors.is_some() { + return config::get_cors(&state, &bucket).await; + } + if query.location.is_some() { + return config::get_location(&state, &bucket).await; + } + if query.encryption.is_some() { + return config::get_encryption(&state, &bucket).await; + } + if query.lifecycle.is_some() { + return config::get_lifecycle(&state, &bucket).await; + } + if query.acl.is_some() { + return config::get_acl(&state, &bucket).await; + } + if query.website.is_some() { + return config::get_website(&state, &bucket).await; + } + if query.object_lock.is_some() { + return config::get_object_lock(&state, &bucket).await; + } + if query.notification.is_some() { + return config::get_notification(&state, &bucket).await; + } + if query.logging.is_some() { + return config::get_logging(&state, &bucket).await; + } + if query.versions.is_some() { + return config::list_object_versions(&state, &bucket).await; + } + if query.uploads.is_some() { + return list_multipart_uploads_handler(&state, &bucket).await; + } + + let prefix = query.prefix.clone().unwrap_or_default(); + let delimiter = query.delimiter.clone().unwrap_or_default(); + let max_keys = query.max_keys.unwrap_or(1000); + + if delimiter.is_empty() { + let params = myfsio_common::types::ListParams { + max_keys, + continuation_token: query.continuation_token.clone(), + prefix: if prefix.is_empty() { None } else { Some(prefix.clone()) }, + start_after: query.start_after.clone(), + }; + match state.storage.list_objects(&bucket, ¶ms).await { + Ok(result) => { + let xml = myfsio_xml::response::list_objects_v2_xml( + &bucket, + &prefix, + &delimiter, + max_keys, + &result.objects, + &[], + result.is_truncated, + query.continuation_token.as_deref(), + result.next_continuation_token.as_deref(), + result.objects.len(), + ); + (StatusCode::OK, [("content-type", "application/xml")], xml).into_response() + } + Err(e) => storage_err_response(e), + } + } else { + let params = myfsio_common::types::ShallowListParams { + prefix, + delimiter: delimiter.clone(), + max_keys, + continuation_token: query.continuation_token.clone(), + }; + match state.storage.list_objects_shallow(&bucket, ¶ms).await { + Ok(result) => { + let xml = myfsio_xml::response::list_objects_v2_xml( + &bucket, + ¶ms.prefix, + &delimiter, + max_keys, + &result.objects, + &result.common_prefixes, + result.is_truncated, + query.continuation_token.as_deref(), + result.next_continuation_token.as_deref(), + result.objects.len() + result.common_prefixes.len(), + ); + (StatusCode::OK, [("content-type", "application/xml")], xml).into_response() + } + Err(e) => storage_err_response(e), + } + } +} + +pub async fn post_bucket( + State(state): State, + Path(bucket): Path, + Query(query): Query, + body: Body, +) -> Response { + if query.delete.is_some() { + return delete_objects_handler(&state, &bucket, body).await; + } + + (StatusCode::METHOD_NOT_ALLOWED).into_response() +} + +pub async fn delete_bucket( + State(state): State, + Path(bucket): Path, + Query(query): Query, +) -> Response { + if query.tagging.is_some() { + return config::delete_tagging(&state, &bucket).await; + } + if query.cors.is_some() { + return config::delete_cors(&state, &bucket).await; + } + if query.encryption.is_some() { + return config::delete_encryption(&state, &bucket).await; + } + if query.lifecycle.is_some() { + return config::delete_lifecycle(&state, &bucket).await; + } + if query.website.is_some() { + return config::delete_website(&state, &bucket).await; + } + + match state.storage.delete_bucket(&bucket).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => storage_err_response(e), + } +} + +pub async fn head_bucket( + State(state): State, + Path(bucket): Path, +) -> Response { + match state.storage.bucket_exists(&bucket).await { + Ok(true) => { + let mut headers = HeaderMap::new(); + headers.insert("x-amz-bucket-region", state.config.region.parse().unwrap()); + (StatusCode::OK, headers).into_response() + } + Ok(false) => storage_err_response( + myfsio_storage::error::StorageError::BucketNotFound(bucket), + ), + Err(e) => storage_err_response(e), + } +} + +#[derive(serde::Deserialize, Default)] +pub struct ObjectQuery { + pub uploads: Option, + #[serde(rename = "uploadId")] + pub upload_id: Option, + #[serde(rename = "partNumber")] + pub part_number: Option, + pub tagging: Option, + pub acl: Option, + pub retention: Option, + #[serde(rename = "legal-hold")] + pub legal_hold: Option, + #[serde(rename = "response-content-type")] + pub response_content_type: Option, + #[serde(rename = "response-content-disposition")] + pub response_content_disposition: Option, + #[serde(rename = "response-content-language")] + pub response_content_language: Option, + #[serde(rename = "response-content-encoding")] + pub response_content_encoding: Option, + #[serde(rename = "response-cache-control")] + pub response_cache_control: Option, + #[serde(rename = "response-expires")] + pub response_expires: Option, +} + +fn apply_response_overrides(headers: &mut HeaderMap, query: &ObjectQuery) { + if let Some(ref v) = query.response_content_type { + if let Ok(val) = v.parse() { headers.insert("content-type", val); } + } + if let Some(ref v) = query.response_content_disposition { + if let Ok(val) = v.parse() { headers.insert("content-disposition", val); } + } + if let Some(ref v) = query.response_content_language { + if let Ok(val) = v.parse() { headers.insert("content-language", val); } + } + if let Some(ref v) = query.response_content_encoding { + if let Ok(val) = v.parse() { headers.insert("content-encoding", val); } + } + if let Some(ref v) = query.response_cache_control { + if let Ok(val) = v.parse() { headers.insert("cache-control", val); } + } + if let Some(ref v) = query.response_expires { + if let Ok(val) = v.parse() { headers.insert("expires", val); } + } +} + +pub async fn put_object( + State(state): State, + Path((bucket, key)): Path<(String, String)>, + Query(query): Query, + headers: HeaderMap, + body: Body, +) -> Response { + if query.tagging.is_some() { + return config::put_object_tagging(&state, &bucket, &key, body).await; + } + if query.acl.is_some() { + return config::put_object_acl(&state, &bucket, &key, body).await; + } + if query.retention.is_some() { + return config::put_object_retention(&state, &bucket, &key, body).await; + } + if query.legal_hold.is_some() { + return config::put_object_legal_hold(&state, &bucket, &key, body).await; + } + + if let Some(ref upload_id) = query.upload_id { + if let Some(part_number) = query.part_number { + return upload_part_handler(&state, &bucket, upload_id, part_number, body).await; + } + } + + if let Some(copy_source) = headers.get("x-amz-copy-source").and_then(|v| v.to_str().ok()) { + return copy_object_handler(&state, copy_source, &bucket, &key).await; + } + + let content_type = headers + .get("content-type") + .and_then(|v| v.to_str().ok()) + .unwrap_or("application/octet-stream"); + + let mut metadata = HashMap::new(); + metadata.insert("__content_type__".to_string(), content_type.to_string()); + + for (name, value) in headers.iter() { + let name_str = name.as_str(); + if let Some(meta_key) = name_str.strip_prefix("x-amz-meta-") { + if let Ok(val) = value.to_str() { + metadata.insert(meta_key.to_string(), val.to_string()); + } + } + } + + let stream = tokio_util::io::StreamReader::new( + http_body_util::BodyStream::new(body).map_ok(|frame| { + frame.into_data().unwrap_or_default() + }).map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)), + ); + let boxed: myfsio_storage::traits::AsyncReadStream = Box::pin(stream); + + match state.storage.put_object(&bucket, &key, boxed, Some(metadata)).await { + Ok(meta) => { + if let Some(enc_ctx) = resolve_encryption_context(&state, &bucket, &headers).await { + if let Some(ref enc_svc) = state.encryption { + let obj_path = match state.storage.get_object_path(&bucket, &key).await { + Ok(p) => p, + Err(e) => return storage_err_response(e), + }; + let tmp_dir = state.config.storage_root.join(".myfsio.sys").join("tmp"); + let _ = tokio::fs::create_dir_all(&tmp_dir).await; + let enc_tmp = tmp_dir.join(format!("enc-{}", uuid::Uuid::new_v4())); + + match enc_svc.encrypt_object(&obj_path, &enc_tmp, &enc_ctx).await { + Ok(enc_meta) => { + if let Err(e) = tokio::fs::rename(&enc_tmp, &obj_path).await { + let _ = tokio::fs::remove_file(&enc_tmp).await; + return storage_err_response(myfsio_storage::error::StorageError::Io(e)); + } + let enc_size = tokio::fs::metadata(&obj_path).await.map(|m| m.len()).unwrap_or(0); + + let mut enc_metadata = enc_meta.to_metadata_map(); + let all_meta = match state.storage.get_object_metadata(&bucket, &key).await { + Ok(m) => m, + Err(_) => HashMap::new(), + }; + for (k, v) in &all_meta { + enc_metadata.entry(k.clone()).or_insert_with(|| v.clone()); + } + enc_metadata.insert("__size__".to_string(), enc_size.to_string()); + let _ = state.storage.put_object_metadata(&bucket, &key, &enc_metadata).await; + + let mut resp_headers = HeaderMap::new(); + if let Some(ref etag) = meta.etag { + resp_headers.insert("etag", format!("\"{}\"", etag).parse().unwrap()); + } + resp_headers.insert("x-amz-server-side-encryption", enc_ctx.algorithm.as_str().parse().unwrap()); + return (StatusCode::OK, resp_headers).into_response(); + } + Err(e) => { + let _ = tokio::fs::remove_file(&enc_tmp).await; + return s3_error_response(S3Error::new( + myfsio_common::error::S3ErrorCode::InternalError, + format!("Encryption failed: {}", e), + )); + } + } + } + } + + let mut resp_headers = HeaderMap::new(); + if let Some(ref etag) = meta.etag { + resp_headers.insert("etag", format!("\"{}\"", etag).parse().unwrap()); + } + (StatusCode::OK, resp_headers).into_response() + } + Err(e) => storage_err_response(e), + } +} + +pub async fn get_object( + State(state): State, + Path((bucket, key)): Path<(String, String)>, + Query(query): Query, + headers: HeaderMap, +) -> Response { + if query.tagging.is_some() { + return config::get_object_tagging(&state, &bucket, &key).await; + } + if query.acl.is_some() { + return config::get_object_acl(&state, &bucket, &key).await; + } + if query.retention.is_some() { + return config::get_object_retention(&state, &bucket, &key).await; + } + if query.legal_hold.is_some() { + return config::get_object_legal_hold(&state, &bucket, &key).await; + } + + let range_header = headers + .get("range") + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()); + + if let Some(ref range_str) = range_header { + return range_get_handler(&state, &bucket, &key, range_str, &query).await; + } + + let all_meta = state.storage.get_object_metadata(&bucket, &key).await.unwrap_or_default(); + let enc_meta = myfsio_crypto::encryption::EncryptionMetadata::from_metadata(&all_meta); + + if let (Some(ref enc_info), Some(ref enc_svc)) = (&enc_meta, &state.encryption) { + let obj_path = match state.storage.get_object_path(&bucket, &key).await { + Ok(p) => p, + Err(e) => return storage_err_response(e), + }; + let tmp_dir = state.config.storage_root.join(".myfsio.sys").join("tmp"); + let _ = tokio::fs::create_dir_all(&tmp_dir).await; + let dec_tmp = tmp_dir.join(format!("dec-{}", uuid::Uuid::new_v4())); + + let customer_key = extract_sse_c_key(&headers); + let ck_ref = customer_key.as_deref(); + + if let Err(e) = enc_svc.decrypt_object(&obj_path, &dec_tmp, enc_info, ck_ref).await { + let _ = tokio::fs::remove_file(&dec_tmp).await; + return s3_error_response(S3Error::new( + myfsio_common::error::S3ErrorCode::InternalError, + format!("Decryption failed: {}", e), + )); + } + + let file = match tokio::fs::File::open(&dec_tmp).await { + Ok(f) => f, + Err(e) => { + let _ = tokio::fs::remove_file(&dec_tmp).await; + return storage_err_response(myfsio_storage::error::StorageError::Io(e)); + } + }; + let file_size = file.metadata().await.map(|m| m.len()).unwrap_or(0); + let stream = ReaderStream::new(file); + let body = Body::from_stream(stream); + + let meta = match state.storage.head_object(&bucket, &key).await { + Ok(m) => m, + Err(e) => { + let _ = tokio::fs::remove_file(&dec_tmp).await; + return storage_err_response(e); + } + }; + + let tmp_path = dec_tmp.clone(); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + let _ = tokio::fs::remove_file(&tmp_path).await; + }); + + let mut resp_headers = HeaderMap::new(); + resp_headers.insert("content-length", file_size.to_string().parse().unwrap()); + if let Some(ref etag) = meta.etag { + resp_headers.insert("etag", format!("\"{}\"", etag).parse().unwrap()); + } + if let Some(ref ct) = meta.content_type { + resp_headers.insert("content-type", ct.parse().unwrap()); + } else { + resp_headers.insert("content-type", "application/octet-stream".parse().unwrap()); + } + resp_headers.insert( + "last-modified", + meta.last_modified.format("%a, %d %b %Y %H:%M:%S GMT").to_string().parse().unwrap(), + ); + resp_headers.insert("accept-ranges", "bytes".parse().unwrap()); + resp_headers.insert("x-amz-server-side-encryption", enc_info.algorithm.parse().unwrap()); + + for (k, v) in &meta.metadata { + if let Ok(header_val) = v.parse() { + let header_name = format!("x-amz-meta-{}", k); + if let Ok(name) = header_name.parse::() { + resp_headers.insert(name, header_val); + } + } + } + + apply_response_overrides(&mut resp_headers, &query); + + return (StatusCode::OK, resp_headers, body).into_response(); + } + + match state.storage.get_object(&bucket, &key).await { + Ok((meta, reader)) => { + let stream = ReaderStream::new(reader); + let body = Body::from_stream(stream); + + let mut headers = HeaderMap::new(); + headers.insert("content-length", meta.size.to_string().parse().unwrap()); + if let Some(ref etag) = meta.etag { + headers.insert("etag", format!("\"{}\"", etag).parse().unwrap()); + } + if let Some(ref ct) = meta.content_type { + headers.insert("content-type", ct.parse().unwrap()); + } else { + headers.insert("content-type", "application/octet-stream".parse().unwrap()); + } + headers.insert( + "last-modified", + meta.last_modified + .format("%a, %d %b %Y %H:%M:%S GMT") + .to_string() + .parse() + .unwrap(), + ); + headers.insert("accept-ranges", "bytes".parse().unwrap()); + + for (k, v) in &meta.metadata { + if let Ok(header_val) = v.parse() { + let header_name = format!("x-amz-meta-{}", k); + if let Ok(name) = header_name.parse::() { + headers.insert(name, header_val); + } + } + } + + apply_response_overrides(&mut headers, &query); + + (StatusCode::OK, headers, body).into_response() + } + Err(e) => storage_err_response(e), + } +} + +pub async fn post_object( + State(state): State, + Path((bucket, key)): Path<(String, String)>, + Query(query): Query, + body: Body, +) -> Response { + if query.uploads.is_some() { + return initiate_multipart_handler(&state, &bucket, &key).await; + } + + if let Some(ref upload_id) = query.upload_id { + return complete_multipart_handler(&state, &bucket, &key, upload_id, body).await; + } + + (StatusCode::METHOD_NOT_ALLOWED).into_response() +} + +pub async fn delete_object( + State(state): State, + Path((bucket, key)): Path<(String, String)>, + Query(query): Query, +) -> Response { + if query.tagging.is_some() { + return config::delete_object_tagging(&state, &bucket, &key).await; + } + + if let Some(ref upload_id) = query.upload_id { + return abort_multipart_handler(&state, &bucket, upload_id).await; + } + + match state.storage.delete_object(&bucket, &key).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => storage_err_response(e), + } +} + +pub async fn head_object( + State(state): State, + Path((bucket, key)): Path<(String, String)>, +) -> Response { + match state.storage.head_object(&bucket, &key).await { + Ok(meta) => { + let mut headers = HeaderMap::new(); + headers.insert("content-length", meta.size.to_string().parse().unwrap()); + if let Some(ref etag) = meta.etag { + headers.insert("etag", format!("\"{}\"", etag).parse().unwrap()); + } + if let Some(ref ct) = meta.content_type { + headers.insert("content-type", ct.parse().unwrap()); + } else { + headers.insert("content-type", "application/octet-stream".parse().unwrap()); + } + headers.insert( + "last-modified", + meta.last_modified + .format("%a, %d %b %Y %H:%M:%S GMT") + .to_string() + .parse() + .unwrap(), + ); + headers.insert("accept-ranges", "bytes".parse().unwrap()); + + for (k, v) in &meta.metadata { + if let Ok(header_val) = v.parse() { + let header_name = format!("x-amz-meta-{}", k); + if let Ok(name) = header_name.parse::() { + headers.insert(name, header_val); + } + } + } + + (StatusCode::OK, headers).into_response() + } + Err(e) => storage_err_response(e), + } +} + +async fn initiate_multipart_handler( + state: &AppState, + bucket: &str, + key: &str, +) -> Response { + match state.storage.initiate_multipart(bucket, key, None).await { + Ok(upload_id) => { + let xml = myfsio_xml::response::initiate_multipart_upload_xml(bucket, key, &upload_id); + (StatusCode::OK, [("content-type", "application/xml")], xml).into_response() + } + Err(e) => storage_err_response(e), + } +} + +async fn upload_part_handler( + state: &AppState, + bucket: &str, + upload_id: &str, + part_number: u32, + body: Body, +) -> Response { + let stream = tokio_util::io::StreamReader::new( + http_body_util::BodyStream::new(body).map_ok(|frame| { + frame.into_data().unwrap_or_default() + }).map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)), + ); + let boxed: myfsio_storage::traits::AsyncReadStream = Box::pin(stream); + + match state.storage.upload_part(bucket, upload_id, part_number, boxed).await { + Ok(etag) => { + let mut headers = HeaderMap::new(); + headers.insert("etag", format!("\"{}\"", etag).parse().unwrap()); + (StatusCode::OK, headers).into_response() + } + Err(e) => storage_err_response(e), + } +} + +async fn complete_multipart_handler( + state: &AppState, + bucket: &str, + key: &str, + upload_id: &str, + body: Body, +) -> Response { + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(collected) => collected.to_bytes(), + Err(_) => { + return s3_error_response(S3Error::new( + myfsio_common::error::S3ErrorCode::MalformedXML, + "Failed to read request body", + )); + } + }; + + let xml_str = String::from_utf8_lossy(&body_bytes); + let parsed = match myfsio_xml::request::parse_complete_multipart_upload(&xml_str) { + Ok(p) => p, + Err(e) => { + return s3_error_response(S3Error::new( + myfsio_common::error::S3ErrorCode::MalformedXML, + e, + )); + } + }; + + let parts: Vec = parsed + .parts + .iter() + .map(|p| PartInfo { + part_number: p.part_number, + etag: p.etag.clone(), + }) + .collect(); + + match state.storage.complete_multipart(bucket, upload_id, &parts).await { + Ok(meta) => { + let etag = meta.etag.as_deref().unwrap_or(""); + let xml = myfsio_xml::response::complete_multipart_upload_xml( + bucket, + key, + etag, + &format!("/{}/{}", bucket, key), + ); + (StatusCode::OK, [("content-type", "application/xml")], xml).into_response() + } + Err(e) => storage_err_response(e), + } +} + +async fn abort_multipart_handler( + state: &AppState, + bucket: &str, + upload_id: &str, +) -> Response { + match state.storage.abort_multipart(bucket, upload_id).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => storage_err_response(e), + } +} + +async fn list_multipart_uploads_handler( + state: &AppState, + bucket: &str, +) -> Response { + match state.storage.list_multipart_uploads(bucket).await { + Ok(uploads) => { + let xml = myfsio_xml::response::list_multipart_uploads_xml(bucket, &uploads); + (StatusCode::OK, [("content-type", "application/xml")], xml).into_response() + } + Err(e) => storage_err_response(e), + } +} + +async fn copy_object_handler( + state: &AppState, + copy_source: &str, + dst_bucket: &str, + dst_key: &str, +) -> Response { + let source = copy_source.strip_prefix('/').unwrap_or(copy_source); + let (src_bucket, src_key) = match source.split_once('/') { + Some(parts) => parts, + None => { + return s3_error_response(S3Error::new( + myfsio_common::error::S3ErrorCode::InvalidArgument, + "Invalid x-amz-copy-source", + )); + } + }; + + match state.storage.copy_object(src_bucket, src_key, dst_bucket, dst_key).await { + Ok(meta) => { + let etag = meta.etag.as_deref().unwrap_or(""); + let last_modified = meta.last_modified.to_rfc3339(); + let xml = myfsio_xml::response::copy_object_result_xml(etag, &last_modified); + (StatusCode::OK, [("content-type", "application/xml")], xml).into_response() + } + Err(e) => storage_err_response(e), + } +} + +async fn delete_objects_handler( + state: &AppState, + bucket: &str, + body: Body, +) -> Response { + let body_bytes = match http_body_util::BodyExt::collect(body).await { + Ok(collected) => collected.to_bytes(), + Err(_) => { + return s3_error_response(S3Error::new( + myfsio_common::error::S3ErrorCode::MalformedXML, + "Failed to read request body", + )); + } + }; + + let xml_str = String::from_utf8_lossy(&body_bytes); + let parsed = match myfsio_xml::request::parse_delete_objects(&xml_str) { + Ok(p) => p, + Err(e) => { + return s3_error_response(S3Error::new( + myfsio_common::error::S3ErrorCode::MalformedXML, + e, + )); + } + }; + + let mut deleted = Vec::new(); + let mut errors = Vec::new(); + + for obj in &parsed.objects { + match state.storage.delete_object(bucket, &obj.key).await { + Ok(()) => deleted.push((obj.key.clone(), obj.version_id.clone())), + Err(e) => { + let s3err = S3Error::from(e); + errors.push(( + obj.key.clone(), + s3err.code.as_str().to_string(), + s3err.message, + )); + } + } + } + + let xml = myfsio_xml::response::delete_result_xml(&deleted, &errors, parsed.quiet); + (StatusCode::OK, [("content-type", "application/xml")], xml).into_response() +} + +async fn range_get_handler( + state: &AppState, + bucket: &str, + key: &str, + range_str: &str, + query: &ObjectQuery, +) -> Response { + let meta = match state.storage.head_object(bucket, key).await { + Ok(m) => m, + Err(e) => return storage_err_response(e), + }; + + let total_size = meta.size; + let (start, end) = match parse_range(range_str, total_size) { + Some(r) => r, + None => { + return s3_error_response(S3Error::new( + myfsio_common::error::S3ErrorCode::InvalidRange, + format!("Range not satisfiable for size {}", total_size), + )); + } + }; + + let path = match state.storage.get_object_path(bucket, key).await { + Ok(p) => p, + Err(e) => return storage_err_response(e), + }; + + let mut file = match tokio::fs::File::open(&path).await { + Ok(f) => f, + Err(e) => return storage_err_response(myfsio_storage::error::StorageError::Io(e)), + }; + + if let Err(e) = file.seek(std::io::SeekFrom::Start(start)).await { + return storage_err_response(myfsio_storage::error::StorageError::Io(e)); + } + + let length = end - start + 1; + let limited = file.take(length); + let stream = ReaderStream::new(limited); + let body = Body::from_stream(stream); + + let mut headers = HeaderMap::new(); + headers.insert("content-length", length.to_string().parse().unwrap()); + headers.insert( + "content-range", + format!("bytes {}-{}/{}", start, end, total_size).parse().unwrap(), + ); + if let Some(ref etag) = meta.etag { + headers.insert("etag", format!("\"{}\"", etag).parse().unwrap()); + } + if let Some(ref ct) = meta.content_type { + headers.insert("content-type", ct.parse().unwrap()); + } else { + headers.insert("content-type", "application/octet-stream".parse().unwrap()); + } + headers.insert("accept-ranges", "bytes".parse().unwrap()); + + apply_response_overrides(&mut headers, query); + + (StatusCode::PARTIAL_CONTENT, headers, body).into_response() +} + +fn parse_range(range_str: &str, total_size: u64) -> Option<(u64, u64)> { + let range_spec = range_str.strip_prefix("bytes=")?; + + if let Some(suffix) = range_spec.strip_prefix('-') { + let suffix_len: u64 = suffix.parse().ok()?; + if suffix_len == 0 || suffix_len > total_size { + return None; + } + return Some((total_size - suffix_len, total_size - 1)); + } + + let (start_str, end_str) = range_spec.split_once('-')?; + let start: u64 = start_str.parse().ok()?; + + let end = if end_str.is_empty() { + total_size - 1 + } else { + let e: u64 = end_str.parse().ok()?; + e.min(total_size - 1) + }; + + if start > end || start >= total_size { + return None; + } + + Some((start, end)) +} + +use futures::TryStreamExt; +use http_body_util; +use tokio::io::AsyncReadExt; + +async fn resolve_encryption_context( + state: &AppState, + bucket: &str, + headers: &HeaderMap, +) -> Option { + if let Some(alg) = headers.get("x-amz-server-side-encryption").and_then(|v| v.to_str().ok()) { + let algorithm = match alg { + "AES256" => myfsio_crypto::encryption::SseAlgorithm::Aes256, + "aws:kms" => myfsio_crypto::encryption::SseAlgorithm::AwsKms, + _ => return None, + }; + let kms_key_id = headers + .get("x-amz-server-side-encryption-aws-kms-key-id") + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()); + return Some(myfsio_crypto::encryption::EncryptionContext { + algorithm, + kms_key_id, + customer_key: None, + }); + } + + if let Some(sse_c_alg) = headers + .get("x-amz-server-side-encryption-customer-algorithm") + .and_then(|v| v.to_str().ok()) + { + if sse_c_alg == "AES256" { + let customer_key = extract_sse_c_key(headers); + if let Some(ck) = customer_key { + return Some(myfsio_crypto::encryption::EncryptionContext { + algorithm: myfsio_crypto::encryption::SseAlgorithm::CustomerProvided, + kms_key_id: None, + customer_key: Some(ck), + }); + } + } + return None; + } + + if state.encryption.is_some() { + if let Ok(config) = state.storage.get_bucket_config(bucket).await { + if let Some(enc_val) = &config.encryption { + let enc_str = enc_val.to_string(); + if enc_str.contains("AES256") { + return Some(myfsio_crypto::encryption::EncryptionContext { + algorithm: myfsio_crypto::encryption::SseAlgorithm::Aes256, + kms_key_id: None, + customer_key: None, + }); + } + if enc_str.contains("aws:kms") { + return Some(myfsio_crypto::encryption::EncryptionContext { + algorithm: myfsio_crypto::encryption::SseAlgorithm::AwsKms, + kms_key_id: None, + customer_key: None, + }); + } + } + } + } + + None +} + +fn extract_sse_c_key(headers: &HeaderMap) -> Option> { + use base64::engine::general_purpose::STANDARD as B64; + use base64::Engine; + + let key_b64 = headers + .get("x-amz-server-side-encryption-customer-key") + .and_then(|v| v.to_str().ok())?; + B64.decode(key_b64).ok() +} diff --git a/myfsio-engine/crates/myfsio-server/src/lib.rs b/myfsio-engine/crates/myfsio-server/src/lib.rs new file mode 100644 index 0000000..47be587 --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/src/lib.rs @@ -0,0 +1,49 @@ +pub mod config; +pub mod handlers; +pub mod middleware; +pub mod services; +pub mod state; + +use axum::Router; + +pub const SERVER_HEADER: &str = concat!("MyFSIO-Rust/", env!("CARGO_PKG_VERSION")); + +pub fn create_router(state: state::AppState) -> Router { + let mut router = Router::new() + .route("/", axum::routing::get(handlers::list_buckets)) + .route( + "/{bucket}", + axum::routing::put(handlers::create_bucket) + .get(handlers::get_bucket) + .delete(handlers::delete_bucket) + .head(handlers::head_bucket) + .post(handlers::post_bucket), + ) + .route( + "/{bucket}/{*key}", + axum::routing::put(handlers::put_object) + .get(handlers::get_object) + .delete(handlers::delete_object) + .head(handlers::head_object) + .post(handlers::post_object), + ); + + if state.config.kms_enabled { + router = router + .route("/kms/keys", axum::routing::get(handlers::kms::list_keys).post(handlers::kms::create_key)) + .route("/kms/keys/{key_id}", axum::routing::get(handlers::kms::get_key).delete(handlers::kms::delete_key)) + .route("/kms/keys/{key_id}/enable", axum::routing::post(handlers::kms::enable_key)) + .route("/kms/keys/{key_id}/disable", axum::routing::post(handlers::kms::disable_key)) + .route("/kms/encrypt", axum::routing::post(handlers::kms::encrypt)) + .route("/kms/decrypt", axum::routing::post(handlers::kms::decrypt)) + .route("/kms/generate-data-key", axum::routing::post(handlers::kms::generate_data_key)); + } + + router + .layer(axum::middleware::from_fn_with_state( + state.clone(), + middleware::auth_layer, + )) + .layer(axum::middleware::from_fn(middleware::server_header)) + .with_state(state) +} diff --git a/myfsio-engine/crates/myfsio-server/src/main.rs b/myfsio-engine/crates/myfsio-server/src/main.rs new file mode 100644 index 0000000..9e2c7c1 --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/src/main.rs @@ -0,0 +1,78 @@ +use myfsio_server::config::ServerConfig; +use myfsio_server::state::AppState; + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt::init(); + + let config = ServerConfig::from_env(); + let bind_addr = config.bind_addr; + + tracing::info!("MyFSIO Rust Engine starting on {}", bind_addr); + tracing::info!("Storage root: {}", config.storage_root.display()); + tracing::info!("Region: {}", config.region); + tracing::info!( + "Encryption: {}, KMS: {}, GC: {}, Lifecycle: {}, Integrity: {}, Metrics: {}", + config.encryption_enabled, + config.kms_enabled, + config.gc_enabled, + config.lifecycle_enabled, + config.integrity_enabled, + config.metrics_enabled + ); + + let state = if config.encryption_enabled || config.kms_enabled { + AppState::new_with_encryption(config.clone()).await + } else { + AppState::new(config.clone()) + }; + + let mut bg_handles: Vec> = Vec::new(); + + if let Some(ref gc) = state.gc { + bg_handles.push(gc.clone().start_background()); + tracing::info!("GC background service started"); + } + + if let Some(ref integrity) = state.integrity { + bg_handles.push(integrity.clone().start_background()); + tracing::info!("Integrity checker background service started"); + } + + if let Some(ref metrics) = state.metrics { + bg_handles.push(metrics.clone().start_background()); + tracing::info!("Metrics collector background service started"); + } + + if config.lifecycle_enabled { + let lifecycle = std::sync::Arc::new( + myfsio_server::services::lifecycle::LifecycleService::new( + state.storage.clone(), + myfsio_server::services::lifecycle::LifecycleConfig::default(), + ), + ); + bg_handles.push(lifecycle.start_background()); + tracing::info!("Lifecycle manager background service started"); + } + + let app = myfsio_server::create_router(state); + + let listener = tokio::net::TcpListener::bind(bind_addr).await.unwrap(); + tracing::info!("Listening on {}", bind_addr); + + axum::serve(listener, app) + .with_graceful_shutdown(shutdown_signal()) + .await + .unwrap(); + + for handle in bg_handles { + handle.abort(); + } +} + +async fn shutdown_signal() { + tokio::signal::ctrl_c() + .await + .expect("Failed to listen for Ctrl+C"); + tracing::info!("Shutdown signal received"); +} diff --git a/myfsio-engine/crates/myfsio-server/src/middleware/auth.rs b/myfsio-engine/crates/myfsio-server/src/middleware/auth.rs new file mode 100644 index 0000000..17ab22e --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/src/middleware/auth.rs @@ -0,0 +1,390 @@ +use axum::extract::{Request, State}; +use axum::http::StatusCode; +use axum::middleware::Next; +use axum::response::{IntoResponse, Response}; + +use chrono::{NaiveDateTime, Utc}; +use myfsio_auth::sigv4; +use myfsio_common::error::{S3Error, S3ErrorCode}; +use myfsio_common::types::Principal; + +use crate::state::AppState; + +pub async fn auth_layer( + State(state): State, + mut req: Request, + next: Next, +) -> Response { + let uri = req.uri().clone(); + let path = uri.path(); + + if path == "/" && req.method() == axum::http::Method::GET { + match try_auth(&state, &req) { + AuthResult::Ok(principal) => { + req.extensions_mut().insert(principal); + } + AuthResult::Denied(err) => return error_response(err), + AuthResult::NoAuth => { + return error_response( + S3Error::from_code(S3ErrorCode::AccessDenied), + ); + } + } + return next.run(req).await; + } + + match try_auth(&state, &req) { + AuthResult::Ok(principal) => { + req.extensions_mut().insert(principal); + next.run(req).await + } + AuthResult::Denied(err) => error_response(err), + AuthResult::NoAuth => { + error_response(S3Error::from_code(S3ErrorCode::AccessDenied)) + } + } +} + +enum AuthResult { + Ok(Principal), + Denied(S3Error), + NoAuth, +} + +fn try_auth(state: &AppState, req: &Request) -> AuthResult { + if let Some(auth_header) = req.headers().get("authorization") { + if let Ok(auth_str) = auth_header.to_str() { + if auth_str.starts_with("AWS4-HMAC-SHA256 ") { + return verify_sigv4_header(state, req, auth_str); + } + } + } + + let query = req.uri().query().unwrap_or(""); + if query.contains("X-Amz-Algorithm=AWS4-HMAC-SHA256") { + return verify_sigv4_query(state, req); + } + + if let (Some(ak), Some(sk)) = ( + req.headers().get("x-access-key").and_then(|v| v.to_str().ok()), + req.headers().get("x-secret-key").and_then(|v| v.to_str().ok()), + ) { + return match state.iam.authenticate(ak, sk) { + Some(principal) => AuthResult::Ok(principal), + None => AuthResult::Denied( + S3Error::from_code(S3ErrorCode::SignatureDoesNotMatch), + ), + }; + } + + AuthResult::NoAuth +} + +fn verify_sigv4_header(state: &AppState, req: &Request, auth_str: &str) -> AuthResult { + let parts: Vec<&str> = auth_str + .strip_prefix("AWS4-HMAC-SHA256 ") + .unwrap() + .split(", ") + .collect(); + + if parts.len() != 3 { + return AuthResult::Denied( + S3Error::new(S3ErrorCode::InvalidArgument, "Malformed Authorization header"), + ); + } + + let credential = parts[0].strip_prefix("Credential=").unwrap_or(""); + let signed_headers_str = parts[1].strip_prefix("SignedHeaders=").unwrap_or(""); + let provided_signature = parts[2].strip_prefix("Signature=").unwrap_or(""); + + let cred_parts: Vec<&str> = credential.split('/').collect(); + if cred_parts.len() != 5 { + return AuthResult::Denied( + S3Error::new(S3ErrorCode::InvalidArgument, "Malformed credential"), + ); + } + + let access_key = cred_parts[0]; + let date_stamp = cred_parts[1]; + let region = cred_parts[2]; + let service = cred_parts[3]; + + let amz_date = req + .headers() + .get("x-amz-date") + .or_else(|| req.headers().get("date")) + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + + if amz_date.is_empty() { + return AuthResult::Denied( + S3Error::new(S3ErrorCode::AccessDenied, "Missing Date header"), + ); + } + + if let Some(err) = check_timestamp_freshness(amz_date, state.config.sigv4_timestamp_tolerance_secs) { + return AuthResult::Denied(err); + } + + let secret_key = match state.iam.get_secret_key(access_key) { + Some(sk) => sk, + None => { + return AuthResult::Denied( + S3Error::from_code(S3ErrorCode::InvalidAccessKeyId), + ); + } + }; + + let method = req.method().as_str(); + let canonical_uri = req.uri().path(); + + let query_params = parse_query_params(req.uri().query().unwrap_or("")); + + let payload_hash = req + .headers() + .get("x-amz-content-sha256") + .and_then(|v| v.to_str().ok()) + .unwrap_or("UNSIGNED-PAYLOAD"); + + let signed_headers: Vec<&str> = signed_headers_str.split(';').collect(); + let header_values: Vec<(String, String)> = signed_headers + .iter() + .map(|&name| { + let value = req + .headers() + .get(name) + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + (name.to_string(), value.to_string()) + }) + .collect(); + + let verified = sigv4::verify_sigv4_signature( + method, + canonical_uri, + &query_params, + signed_headers_str, + &header_values, + payload_hash, + amz_date, + date_stamp, + region, + service, + &secret_key, + provided_signature, + ); + + if !verified { + return AuthResult::Denied( + S3Error::from_code(S3ErrorCode::SignatureDoesNotMatch), + ); + } + + match state.iam.get_principal(access_key) { + Some(p) => AuthResult::Ok(p), + None => AuthResult::Denied( + S3Error::from_code(S3ErrorCode::InvalidAccessKeyId), + ), + } +} + +fn verify_sigv4_query(state: &AppState, req: &Request) -> AuthResult { + let query = req.uri().query().unwrap_or(""); + let params = parse_query_params(query); + let param_map: std::collections::HashMap<&str, &str> = params + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect(); + + let credential = match param_map.get("X-Amz-Credential") { + Some(c) => *c, + None => { + return AuthResult::Denied( + S3Error::new(S3ErrorCode::InvalidArgument, "Missing X-Amz-Credential"), + ); + } + }; + + let signed_headers_str = param_map + .get("X-Amz-SignedHeaders") + .copied() + .unwrap_or("host"); + let provided_signature = match param_map.get("X-Amz-Signature") { + Some(s) => *s, + None => { + return AuthResult::Denied( + S3Error::new(S3ErrorCode::InvalidArgument, "Missing X-Amz-Signature"), + ); + } + }; + let amz_date = match param_map.get("X-Amz-Date") { + Some(d) => *d, + None => { + return AuthResult::Denied( + S3Error::new(S3ErrorCode::InvalidArgument, "Missing X-Amz-Date"), + ); + } + }; + let expires_str = match param_map.get("X-Amz-Expires") { + Some(e) => *e, + None => { + return AuthResult::Denied( + S3Error::new(S3ErrorCode::InvalidArgument, "Missing X-Amz-Expires"), + ); + } + }; + + let cred_parts: Vec<&str> = credential.split('/').collect(); + if cred_parts.len() != 5 { + return AuthResult::Denied( + S3Error::new(S3ErrorCode::InvalidArgument, "Malformed credential"), + ); + } + + let access_key = cred_parts[0]; + let date_stamp = cred_parts[1]; + let region = cred_parts[2]; + let service = cred_parts[3]; + + let expires: u64 = match expires_str.parse() { + Ok(e) => e, + Err(_) => { + return AuthResult::Denied( + S3Error::new(S3ErrorCode::InvalidArgument, "Invalid X-Amz-Expires"), + ); + } + }; + + if expires < state.config.presigned_url_min_expiry + || expires > state.config.presigned_url_max_expiry + { + return AuthResult::Denied( + S3Error::new(S3ErrorCode::InvalidArgument, "X-Amz-Expires out of range"), + ); + } + + if let Ok(request_time) = + NaiveDateTime::parse_from_str(amz_date, "%Y%m%dT%H%M%SZ") + { + let request_utc = request_time.and_utc(); + let now = Utc::now(); + let elapsed = (now - request_utc).num_seconds(); + if elapsed > expires as i64 { + return AuthResult::Denied( + S3Error::new(S3ErrorCode::AccessDenied, "Request has expired"), + ); + } + if elapsed < -(state.config.sigv4_timestamp_tolerance_secs as i64) { + return AuthResult::Denied( + S3Error::new(S3ErrorCode::AccessDenied, "Request is too far in the future"), + ); + } + } + + let secret_key = match state.iam.get_secret_key(access_key) { + Some(sk) => sk, + None => { + return AuthResult::Denied( + S3Error::from_code(S3ErrorCode::InvalidAccessKeyId), + ); + } + }; + + let method = req.method().as_str(); + let canonical_uri = req.uri().path(); + + let query_params_no_sig: Vec<(String, String)> = params + .iter() + .filter(|(k, _)| k != "X-Amz-Signature") + .cloned() + .collect(); + + let payload_hash = "UNSIGNED-PAYLOAD"; + + let signed_headers: Vec<&str> = signed_headers_str.split(';').collect(); + let header_values: Vec<(String, String)> = signed_headers + .iter() + .map(|&name| { + let value = req + .headers() + .get(name) + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + (name.to_string(), value.to_string()) + }) + .collect(); + + let verified = sigv4::verify_sigv4_signature( + method, + canonical_uri, + &query_params_no_sig, + signed_headers_str, + &header_values, + payload_hash, + amz_date, + date_stamp, + region, + service, + &secret_key, + provided_signature, + ); + + if !verified { + return AuthResult::Denied( + S3Error::from_code(S3ErrorCode::SignatureDoesNotMatch), + ); + } + + match state.iam.get_principal(access_key) { + Some(p) => AuthResult::Ok(p), + None => AuthResult::Denied( + S3Error::from_code(S3ErrorCode::InvalidAccessKeyId), + ), + } +} + +fn check_timestamp_freshness(amz_date: &str, tolerance_secs: u64) -> Option { + let request_time = NaiveDateTime::parse_from_str(amz_date, "%Y%m%dT%H%M%SZ").ok()?; + let request_utc = request_time.and_utc(); + let now = Utc::now(); + let diff = (now - request_utc).num_seconds().unsigned_abs(); + + if diff > tolerance_secs { + return Some(S3Error::new( + S3ErrorCode::AccessDenied, + "Request timestamp too old or too far in the future", + )); + } + None +} + +fn parse_query_params(query: &str) -> Vec<(String, String)> { + if query.is_empty() { + return Vec::new(); + } + query + .split('&') + .filter_map(|pair| { + let mut parts = pair.splitn(2, '='); + let key = parts.next()?; + let value = parts.next().unwrap_or(""); + Some(( + urlencoding_decode(key), + urlencoding_decode(value), + )) + }) + .collect() +} + +fn urlencoding_decode(s: &str) -> String { + percent_encoding::percent_decode_str(s) + .decode_utf8_lossy() + .into_owned() +} + +fn error_response(err: S3Error) -> Response { + let status = + StatusCode::from_u16(err.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + let body = err.to_xml(); + (status, [("content-type", "application/xml")], body).into_response() +} diff --git a/myfsio-engine/crates/myfsio-server/src/middleware/mod.rs b/myfsio-engine/crates/myfsio-server/src/middleware/mod.rs new file mode 100644 index 0000000..9d380ad --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/src/middleware/mod.rs @@ -0,0 +1,16 @@ +mod auth; + +pub use auth::auth_layer; + +use axum::extract::Request; +use axum::middleware::Next; +use axum::response::Response; + +pub async fn server_header(req: Request, next: Next) -> Response { + let mut resp = next.run(req).await; + resp.headers_mut().insert( + "server", + crate::SERVER_HEADER.parse().unwrap(), + ); + resp +} diff --git a/myfsio-engine/crates/myfsio-server/src/services/gc.rs b/myfsio-engine/crates/myfsio-server/src/services/gc.rs new file mode 100644 index 0000000..e1d930f --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/src/services/gc.rs @@ -0,0 +1,263 @@ +use serde_json::{json, Value}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Instant; +use tokio::sync::RwLock; + +pub struct GcConfig { + pub interval_hours: f64, + pub temp_file_max_age_hours: f64, + pub multipart_max_age_days: u64, + pub lock_file_max_age_hours: f64, + pub dry_run: bool, +} + +impl Default for GcConfig { + fn default() -> Self { + Self { + interval_hours: 6.0, + temp_file_max_age_hours: 24.0, + multipart_max_age_days: 7, + lock_file_max_age_hours: 1.0, + dry_run: false, + } + } +} + +pub struct GcService { + storage_root: PathBuf, + config: GcConfig, + running: Arc>, + history: Arc>>, + history_path: PathBuf, +} + +impl GcService { + pub fn new(storage_root: PathBuf, config: GcConfig) -> Self { + let history_path = storage_root + .join(".myfsio.sys") + .join("config") + .join("gc_history.json"); + + let history = if history_path.exists() { + std::fs::read_to_string(&history_path) + .ok() + .and_then(|s| serde_json::from_str::(&s).ok()) + .and_then(|v| v.get("executions").and_then(|e| e.as_array().cloned())) + .unwrap_or_default() + } else { + Vec::new() + }; + + Self { + storage_root, + config, + running: Arc::new(RwLock::new(false)), + history: Arc::new(RwLock::new(history)), + history_path, + } + } + + pub async fn status(&self) -> Value { + let running = *self.running.read().await; + json!({ + "enabled": true, + "running": running, + "interval_hours": self.config.interval_hours, + "temp_file_max_age_hours": self.config.temp_file_max_age_hours, + "multipart_max_age_days": self.config.multipart_max_age_days, + "lock_file_max_age_hours": self.config.lock_file_max_age_hours, + "dry_run": self.config.dry_run, + }) + } + + pub async fn history(&self) -> Value { + let history = self.history.read().await; + json!({ "executions": *history }) + } + + pub async fn run_now(&self, dry_run: bool) -> Result { + { + let mut running = self.running.write().await; + if *running { + return Err("GC already running".to_string()); + } + *running = true; + } + + let start = Instant::now(); + let result = self.execute_gc(dry_run || self.config.dry_run).await; + let elapsed = start.elapsed().as_secs_f64(); + + *self.running.write().await = false; + + let mut result_json = result.clone(); + if let Some(obj) = result_json.as_object_mut() { + obj.insert("execution_time_seconds".to_string(), json!(elapsed)); + } + + let record = json!({ + "timestamp": chrono::Utc::now().timestamp_millis() as f64 / 1000.0, + "dry_run": dry_run || self.config.dry_run, + "result": result_json, + }); + + { + let mut history = self.history.write().await; + history.push(record); + if history.len() > 50 { + let excess = history.len() - 50; + history.drain(..excess); + } + } + self.save_history().await; + + Ok(result) + } + + async fn execute_gc(&self, dry_run: bool) -> Value { + let mut temp_files_deleted = 0u64; + let mut temp_bytes_freed = 0u64; + let mut multipart_uploads_deleted = 0u64; + let mut lock_files_deleted = 0u64; + let mut empty_dirs_removed = 0u64; + let mut errors: Vec = Vec::new(); + + let now = std::time::SystemTime::now(); + let temp_max_age = std::time::Duration::from_secs_f64(self.config.temp_file_max_age_hours * 3600.0); + let multipart_max_age = std::time::Duration::from_secs(self.config.multipart_max_age_days * 86400); + let lock_max_age = std::time::Duration::from_secs_f64(self.config.lock_file_max_age_hours * 3600.0); + + let tmp_dir = self.storage_root.join(".myfsio.sys").join("tmp"); + if tmp_dir.exists() { + match std::fs::read_dir(&tmp_dir) { + Ok(entries) => { + for entry in entries.flatten() { + if let Ok(metadata) = entry.metadata() { + if let Ok(modified) = metadata.modified() { + if let Ok(age) = now.duration_since(modified) { + if age > temp_max_age { + let size = metadata.len(); + if !dry_run { + if let Err(e) = std::fs::remove_file(entry.path()) { + errors.push(format!("Failed to remove temp file: {}", e)); + continue; + } + } + temp_files_deleted += 1; + temp_bytes_freed += size; + } + } + } + } + } + } + Err(e) => errors.push(format!("Failed to read tmp dir: {}", e)), + } + } + + let multipart_dir = self.storage_root.join(".myfsio.sys").join("multipart"); + if multipart_dir.exists() { + if let Ok(bucket_dirs) = std::fs::read_dir(&multipart_dir) { + for bucket_entry in bucket_dirs.flatten() { + if let Ok(uploads) = std::fs::read_dir(bucket_entry.path()) { + for upload in uploads.flatten() { + if let Ok(metadata) = upload.metadata() { + if let Ok(modified) = metadata.modified() { + if let Ok(age) = now.duration_since(modified) { + if age > multipart_max_age { + if !dry_run { + let _ = std::fs::remove_dir_all(upload.path()); + } + multipart_uploads_deleted += 1; + } + } + } + } + } + } + } + } + } + + let buckets_dir = self.storage_root.join(".myfsio.sys").join("buckets"); + if buckets_dir.exists() { + if let Ok(bucket_dirs) = std::fs::read_dir(&buckets_dir) { + for bucket_entry in bucket_dirs.flatten() { + let locks_dir = bucket_entry.path().join("locks"); + if locks_dir.exists() { + if let Ok(locks) = std::fs::read_dir(&locks_dir) { + for lock in locks.flatten() { + if let Ok(metadata) = lock.metadata() { + if let Ok(modified) = metadata.modified() { + if let Ok(age) = now.duration_since(modified) { + if age > lock_max_age { + if !dry_run { + let _ = std::fs::remove_file(lock.path()); + } + lock_files_deleted += 1; + } + } + } + } + } + } + } + } + } + } + + if !dry_run { + for dir in [&tmp_dir, &multipart_dir] { + if dir.exists() { + if let Ok(entries) = std::fs::read_dir(dir) { + for entry in entries.flatten() { + if entry.path().is_dir() { + if let Ok(mut contents) = std::fs::read_dir(entry.path()) { + if contents.next().is_none() { + let _ = std::fs::remove_dir(entry.path()); + empty_dirs_removed += 1; + } + } + } + } + } + } + } + } + + json!({ + "temp_files_deleted": temp_files_deleted, + "temp_bytes_freed": temp_bytes_freed, + "multipart_uploads_deleted": multipart_uploads_deleted, + "lock_files_deleted": lock_files_deleted, + "empty_dirs_removed": empty_dirs_removed, + "errors": errors, + }) + } + + async fn save_history(&self) { + let history = self.history.read().await; + let data = json!({ "executions": *history }); + if let Some(parent) = self.history_path.parent() { + let _ = std::fs::create_dir_all(parent); + } + let _ = std::fs::write(&self.history_path, serde_json::to_string_pretty(&data).unwrap_or_default()); + } + + pub fn start_background(self: Arc) -> tokio::task::JoinHandle<()> { + let interval = std::time::Duration::from_secs_f64(self.config.interval_hours * 3600.0); + tokio::spawn(async move { + let mut timer = tokio::time::interval(interval); + timer.tick().await; + loop { + timer.tick().await; + tracing::info!("GC cycle starting"); + match self.run_now(false).await { + Ok(result) => tracing::info!("GC cycle complete: {:?}", result), + Err(e) => tracing::warn!("GC cycle failed: {}", e), + } + } + }) + } +} diff --git a/myfsio-engine/crates/myfsio-server/src/services/integrity.rs b/myfsio-engine/crates/myfsio-server/src/services/integrity.rs new file mode 100644 index 0000000..7cf2f03 --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/src/services/integrity.rs @@ -0,0 +1,204 @@ +use myfsio_storage::fs_backend::FsStorageBackend; +use myfsio_storage::traits::StorageEngine; +use serde_json::{json, Value}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Instant; +use tokio::sync::RwLock; + +pub struct IntegrityConfig { + pub interval_hours: f64, + pub batch_size: usize, + pub auto_heal: bool, + pub dry_run: bool, +} + +impl Default for IntegrityConfig { + fn default() -> Self { + Self { + interval_hours: 24.0, + batch_size: 1000, + auto_heal: false, + dry_run: false, + } + } +} + +pub struct IntegrityService { + storage: Arc, + config: IntegrityConfig, + running: Arc>, + history: Arc>>, + history_path: PathBuf, +} + +impl IntegrityService { + pub fn new( + storage: Arc, + storage_root: &std::path::Path, + config: IntegrityConfig, + ) -> Self { + let history_path = storage_root + .join(".myfsio.sys") + .join("config") + .join("integrity_history.json"); + + let history = if history_path.exists() { + std::fs::read_to_string(&history_path) + .ok() + .and_then(|s| serde_json::from_str::(&s).ok()) + .and_then(|v| v.get("executions").and_then(|e| e.as_array().cloned())) + .unwrap_or_default() + } else { + Vec::new() + }; + + Self { + storage, + config, + running: Arc::new(RwLock::new(false)), + history: Arc::new(RwLock::new(history)), + history_path, + } + } + + pub async fn status(&self) -> Value { + let running = *self.running.read().await; + json!({ + "enabled": true, + "running": running, + "interval_hours": self.config.interval_hours, + "batch_size": self.config.batch_size, + "auto_heal": self.config.auto_heal, + "dry_run": self.config.dry_run, + }) + } + + pub async fn history(&self) -> Value { + let history = self.history.read().await; + json!({ "executions": *history }) + } + + pub async fn run_now(&self, dry_run: bool, auto_heal: bool) -> Result { + { + let mut running = self.running.write().await; + if *running { + return Err("Integrity check already running".to_string()); + } + *running = true; + } + + let start = Instant::now(); + let result = self.check_integrity(dry_run, auto_heal).await; + let elapsed = start.elapsed().as_secs_f64(); + + *self.running.write().await = false; + + let mut result_json = result.clone(); + if let Some(obj) = result_json.as_object_mut() { + obj.insert("execution_time_seconds".to_string(), json!(elapsed)); + } + + let record = json!({ + "timestamp": chrono::Utc::now().timestamp_millis() as f64 / 1000.0, + "dry_run": dry_run, + "auto_heal": auto_heal, + "result": result_json, + }); + + { + let mut history = self.history.write().await; + history.push(record); + if history.len() > 50 { + let excess = history.len() - 50; + history.drain(..excess); + } + } + self.save_history().await; + + Ok(result) + } + + async fn check_integrity(&self, _dry_run: bool, _auto_heal: bool) -> Value { + let buckets = match self.storage.list_buckets().await { + Ok(b) => b, + Err(e) => return json!({"error": e.to_string()}), + }; + + let mut objects_scanned = 0u64; + let mut corrupted = 0u64; + let mut phantom_metadata = 0u64; + let mut errors: Vec = Vec::new(); + + for bucket in &buckets { + let params = myfsio_common::types::ListParams { + max_keys: self.config.batch_size, + ..Default::default() + }; + let objects = match self.storage.list_objects(&bucket.name, ¶ms).await { + Ok(r) => r.objects, + Err(e) => { + errors.push(format!("{}: {}", bucket.name, e)); + continue; + } + }; + + for obj in &objects { + objects_scanned += 1; + match self.storage.get_object_path(&bucket.name, &obj.key).await { + Ok(path) => { + if !path.exists() { + phantom_metadata += 1; + } else if let Some(ref expected_etag) = obj.etag { + match myfsio_crypto::hashing::md5_file(&path) { + Ok(actual_etag) => { + if &actual_etag != expected_etag { + corrupted += 1; + } + } + Err(e) => errors.push(format!("{}:{}: {}", bucket.name, obj.key, e)), + } + } + } + Err(e) => errors.push(format!("{}:{}: {}", bucket.name, obj.key, e)), + } + } + } + + json!({ + "objects_scanned": objects_scanned, + "buckets_scanned": buckets.len(), + "corrupted_objects": corrupted, + "phantom_metadata": phantom_metadata, + "errors": errors, + }) + } + + async fn save_history(&self) { + let history = self.history.read().await; + let data = json!({ "executions": *history }); + if let Some(parent) = self.history_path.parent() { + let _ = std::fs::create_dir_all(parent); + } + let _ = std::fs::write( + &self.history_path, + serde_json::to_string_pretty(&data).unwrap_or_default(), + ); + } + + pub fn start_background(self: Arc) -> tokio::task::JoinHandle<()> { + let interval = std::time::Duration::from_secs_f64(self.config.interval_hours * 3600.0); + tokio::spawn(async move { + let mut timer = tokio::time::interval(interval); + timer.tick().await; + loop { + timer.tick().await; + tracing::info!("Integrity check starting"); + match self.run_now(false, false).await { + Ok(result) => tracing::info!("Integrity check complete: {:?}", result), + Err(e) => tracing::warn!("Integrity check failed: {}", e), + } + } + }) + } +} diff --git a/myfsio-engine/crates/myfsio-server/src/services/lifecycle.rs b/myfsio-engine/crates/myfsio-server/src/services/lifecycle.rs new file mode 100644 index 0000000..b68706a --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/src/services/lifecycle.rs @@ -0,0 +1,153 @@ +use myfsio_storage::fs_backend::FsStorageBackend; +use myfsio_storage::traits::StorageEngine; +use serde_json::{json, Value}; +use std::sync::Arc; +use tokio::sync::RwLock; + +pub struct LifecycleConfig { + pub interval_seconds: u64, +} + +impl Default for LifecycleConfig { + fn default() -> Self { + Self { + interval_seconds: 3600, + } + } +} + +pub struct LifecycleService { + storage: Arc, + config: LifecycleConfig, + running: Arc>, +} + +impl LifecycleService { + pub fn new(storage: Arc, config: LifecycleConfig) -> Self { + Self { + storage, + config, + running: Arc::new(RwLock::new(false)), + } + } + + pub async fn run_cycle(&self) -> Result { + { + let mut running = self.running.write().await; + if *running { + return Err("Lifecycle already running".to_string()); + } + *running = true; + } + + let result = self.evaluate_rules().await; + *self.running.write().await = false; + Ok(result) + } + + async fn evaluate_rules(&self) -> Value { + let buckets = match self.storage.list_buckets().await { + Ok(b) => b, + Err(e) => return json!({"error": e.to_string()}), + }; + + let mut total_expired = 0u64; + let mut total_multipart_aborted = 0u64; + let mut errors: Vec = Vec::new(); + + for bucket in &buckets { + let config = match self.storage.get_bucket_config(&bucket.name).await { + Ok(c) => c, + Err(_) => continue, + }; + + let lifecycle = match &config.lifecycle { + Some(lc) => lc, + None => continue, + }; + + let rules = match lifecycle.as_str().and_then(|s| serde_json::from_str::(s).ok()) { + Some(v) => v, + None => continue, + }; + + let rules_arr = match rules.get("Rules").and_then(|r| r.as_array()) { + Some(a) => a.clone(), + None => continue, + }; + + for rule in &rules_arr { + if rule.get("Status").and_then(|s| s.as_str()) != Some("Enabled") { + continue; + } + + let prefix = rule + .get("Filter") + .and_then(|f| f.get("Prefix")) + .and_then(|p| p.as_str()) + .or_else(|| rule.get("Prefix").and_then(|p| p.as_str())) + .unwrap_or(""); + + if let Some(exp) = rule.get("Expiration") { + if let Some(days) = exp.get("Days").and_then(|d| d.as_u64()) { + let cutoff = chrono::Utc::now() - chrono::Duration::days(days as i64); + let params = myfsio_common::types::ListParams { + max_keys: 1000, + prefix: if prefix.is_empty() { None } else { Some(prefix.to_string()) }, + ..Default::default() + }; + if let Ok(result) = self.storage.list_objects(&bucket.name, ¶ms).await { + for obj in &result.objects { + if obj.last_modified < cutoff { + match self.storage.delete_object(&bucket.name, &obj.key).await { + Ok(()) => total_expired += 1, + Err(e) => errors.push(format!("{}:{}: {}", bucket.name, obj.key, e)), + } + } + } + } + } + } + + if let Some(abort) = rule.get("AbortIncompleteMultipartUpload") { + if let Some(days) = abort.get("DaysAfterInitiation").and_then(|d| d.as_u64()) { + let cutoff = chrono::Utc::now() - chrono::Duration::days(days as i64); + if let Ok(uploads) = self.storage.list_multipart_uploads(&bucket.name).await { + for upload in &uploads { + if upload.initiated < cutoff { + match self.storage.abort_multipart(&bucket.name, &upload.upload_id).await { + Ok(()) => total_multipart_aborted += 1, + Err(e) => errors.push(format!("abort {}: {}", upload.upload_id, e)), + } + } + } + } + } + } + } + } + + json!({ + "objects_expired": total_expired, + "multipart_aborted": total_multipart_aborted, + "buckets_evaluated": buckets.len(), + "errors": errors, + }) + } + + pub fn start_background(self: Arc) -> tokio::task::JoinHandle<()> { + let interval = std::time::Duration::from_secs(self.config.interval_seconds); + tokio::spawn(async move { + let mut timer = tokio::time::interval(interval); + timer.tick().await; + loop { + timer.tick().await; + tracing::info!("Lifecycle evaluation starting"); + match self.run_cycle().await { + Ok(result) => tracing::info!("Lifecycle cycle complete: {:?}", result), + Err(e) => tracing::warn!("Lifecycle cycle failed: {}", e), + } + } + }) + } +} diff --git a/myfsio-engine/crates/myfsio-server/src/services/metrics.rs b/myfsio-engine/crates/myfsio-server/src/services/metrics.rs new file mode 100644 index 0000000..c95fe03 --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/src/services/metrics.rs @@ -0,0 +1,219 @@ +use serde_json::{json, Value}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Instant; +use tokio::sync::RwLock; + +pub struct MetricsConfig { + pub interval_minutes: u64, + pub retention_hours: u64, +} + +impl Default for MetricsConfig { + fn default() -> Self { + Self { + interval_minutes: 5, + retention_hours: 24, + } + } +} + +struct MethodStats { + count: u64, + success_count: u64, + error_count: u64, + bytes_in: u64, + bytes_out: u64, + latencies: Vec, +} + +impl MethodStats { + fn new() -> Self { + Self { + count: 0, + success_count: 0, + error_count: 0, + bytes_in: 0, + bytes_out: 0, + latencies: Vec::new(), + } + } + + fn to_json(&self) -> Value { + let (min, max, avg, p50, p95, p99) = if self.latencies.is_empty() { + (0.0, 0.0, 0.0, 0.0, 0.0, 0.0) + } else { + let mut sorted = self.latencies.clone(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let len = sorted.len(); + let sum: f64 = sorted.iter().sum(); + ( + sorted[0], + sorted[len - 1], + sum / len as f64, + sorted[len / 2], + sorted[((len as f64 * 0.95) as usize).min(len - 1)], + sorted[((len as f64 * 0.99) as usize).min(len - 1)], + ) + }; + + json!({ + "count": self.count, + "success_count": self.success_count, + "error_count": self.error_count, + "bytes_in": self.bytes_in, + "bytes_out": self.bytes_out, + "latency_min_ms": min, + "latency_max_ms": max, + "latency_avg_ms": avg, + "latency_p50_ms": p50, + "latency_p95_ms": p95, + "latency_p99_ms": p99, + }) + } +} + +struct CurrentWindow { + by_method: HashMap, + by_status_class: HashMap, + start_time: Instant, +} + +impl CurrentWindow { + fn new() -> Self { + Self { + by_method: HashMap::new(), + by_status_class: HashMap::new(), + start_time: Instant::now(), + } + } + + fn reset(&mut self) { + self.by_method.clear(); + self.by_status_class.clear(); + self.start_time = Instant::now(); + } +} + +pub struct MetricsService { + config: MetricsConfig, + current: Arc>, + snapshots: Arc>>, + snapshots_path: PathBuf, +} + +impl MetricsService { + pub fn new(storage_root: &std::path::Path, config: MetricsConfig) -> Self { + let snapshots_path = storage_root + .join(".myfsio.sys") + .join("config") + .join("operation_metrics.json"); + + let snapshots = if snapshots_path.exists() { + std::fs::read_to_string(&snapshots_path) + .ok() + .and_then(|s| serde_json::from_str::(&s).ok()) + .and_then(|v| v.get("snapshots").and_then(|s| s.as_array().cloned())) + .unwrap_or_default() + } else { + Vec::new() + }; + + Self { + config, + current: Arc::new(RwLock::new(CurrentWindow::new())), + snapshots: Arc::new(RwLock::new(snapshots)), + snapshots_path, + } + } + + pub async fn record(&self, method: &str, status: u16, latency_ms: f64, bytes_in: u64, bytes_out: u64) { + let mut window = self.current.write().await; + let stats = window.by_method.entry(method.to_string()).or_insert_with(MethodStats::new); + stats.count += 1; + if status < 400 { + stats.success_count += 1; + } else { + stats.error_count += 1; + } + stats.bytes_in += bytes_in; + stats.bytes_out += bytes_out; + stats.latencies.push(latency_ms); + + let class = format!("{}xx", status / 100); + *window.by_status_class.entry(class).or_insert(0) += 1; + } + + pub async fn snapshot(&self) -> Value { + let window = self.current.read().await; + let mut by_method = serde_json::Map::new(); + for (method, stats) in &window.by_method { + by_method.insert(method.clone(), stats.to_json()); + } + + let snapshots = self.snapshots.read().await; + json!({ + "enabled": true, + "current_window": { + "by_method": by_method, + "by_status_class": window.by_status_class, + "window_start_elapsed_secs": window.start_time.elapsed().as_secs_f64(), + }, + "snapshots": *snapshots, + }) + } + + async fn flush_window(&self) { + let snap = { + let mut window = self.current.write().await; + let mut by_method = serde_json::Map::new(); + for (method, stats) in &window.by_method { + by_method.insert(method.clone(), stats.to_json()); + } + let snap = json!({ + "timestamp": chrono::Utc::now().to_rfc3339(), + "window_seconds": self.config.interval_minutes * 60, + "by_method": by_method, + "by_status_class": window.by_status_class, + }); + window.reset(); + snap + }; + + let max_snapshots = (self.config.retention_hours * 60 / self.config.interval_minutes) as usize; + { + let mut snapshots = self.snapshots.write().await; + snapshots.push(snap); + if snapshots.len() > max_snapshots { + let excess = snapshots.len() - max_snapshots; + snapshots.drain(..excess); + } + } + self.save_snapshots().await; + } + + async fn save_snapshots(&self) { + let snapshots = self.snapshots.read().await; + let data = json!({ "snapshots": *snapshots }); + if let Some(parent) = self.snapshots_path.parent() { + let _ = std::fs::create_dir_all(parent); + } + let _ = std::fs::write( + &self.snapshots_path, + serde_json::to_string_pretty(&data).unwrap_or_default(), + ); + } + + pub fn start_background(self: Arc) -> tokio::task::JoinHandle<()> { + let interval = std::time::Duration::from_secs(self.config.interval_minutes * 60); + tokio::spawn(async move { + let mut timer = tokio::time::interval(interval); + timer.tick().await; + loop { + timer.tick().await; + self.flush_window().await; + } + }) + } +} diff --git a/myfsio-engine/crates/myfsio-server/src/services/mod.rs b/myfsio-engine/crates/myfsio-server/src/services/mod.rs new file mode 100644 index 0000000..08a673c --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/src/services/mod.rs @@ -0,0 +1,4 @@ +pub mod gc; +pub mod lifecycle; +pub mod integrity; +pub mod metrics; diff --git a/myfsio-engine/crates/myfsio-server/src/state.rs b/myfsio-engine/crates/myfsio-server/src/state.rs new file mode 100644 index 0000000..945a061 --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/src/state.rs @@ -0,0 +1,107 @@ +use std::sync::Arc; + +use crate::config::ServerConfig; +use crate::services::gc::GcService; +use crate::services::integrity::IntegrityService; +use crate::services::metrics::MetricsService; +use myfsio_auth::iam::IamService; +use myfsio_crypto::encryption::EncryptionService; +use myfsio_crypto::kms::KmsService; +use myfsio_storage::fs_backend::FsStorageBackend; + +#[derive(Clone)] +pub struct AppState { + pub config: ServerConfig, + pub storage: Arc, + pub iam: Arc, + pub encryption: Option>, + pub kms: Option>, + pub gc: Option>, + pub integrity: Option>, + pub metrics: Option>, +} + +impl AppState { + pub fn new(config: ServerConfig) -> Self { + let storage = Arc::new(FsStorageBackend::new(config.storage_root.clone())); + let iam = Arc::new(IamService::new_with_secret( + config.iam_config_path.clone(), + config.secret_key.clone(), + )); + + let gc = if config.gc_enabled { + Some(Arc::new(GcService::new( + config.storage_root.clone(), + crate::services::gc::GcConfig::default(), + ))) + } else { + None + }; + + let integrity = if config.integrity_enabled { + Some(Arc::new(IntegrityService::new( + storage.clone(), + &config.storage_root, + crate::services::integrity::IntegrityConfig::default(), + ))) + } else { + None + }; + + let metrics = if config.metrics_enabled { + Some(Arc::new(MetricsService::new( + &config.storage_root, + crate::services::metrics::MetricsConfig::default(), + ))) + } else { + None + }; + + Self { + config, + storage, + iam, + encryption: None, + kms: None, + gc, + integrity, + metrics, + } + } + + pub async fn new_with_encryption(config: ServerConfig) -> Self { + let mut state = Self::new(config.clone()); + + let keys_dir = config.storage_root.join(".myfsio.sys").join("keys"); + + let kms = if config.kms_enabled { + match KmsService::new(&keys_dir).await { + Ok(k) => Some(Arc::new(k)), + Err(e) => { + tracing::error!("Failed to initialize KMS: {}", e); + None + } + } + } else { + None + }; + + let encryption = if config.encryption_enabled { + match myfsio_crypto::kms::load_or_create_master_key(&keys_dir).await { + Ok(master_key) => { + Some(Arc::new(EncryptionService::new(master_key, kms.clone()))) + } + Err(e) => { + tracing::error!("Failed to initialize encryption: {}", e); + None + } + } + } else { + None + }; + + state.encryption = encryption; + state.kms = kms; + state + } +} diff --git a/myfsio-engine/crates/myfsio-server/tests/integration.rs b/myfsio-engine/crates/myfsio-server/tests/integration.rs new file mode 100644 index 0000000..28d21cc --- /dev/null +++ b/myfsio-engine/crates/myfsio-server/tests/integration.rs @@ -0,0 +1,1413 @@ +use axum::body::Body; +use axum::http::{Method, Request, StatusCode}; +use http_body_util::BodyExt; +use tower::ServiceExt; + +const TEST_ACCESS_KEY: &str = "AKIAIOSFODNN7EXAMPLE"; +const TEST_SECRET_KEY: &str = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + +fn test_app() -> (axum::Router, tempfile::TempDir) { + let tmp = tempfile::TempDir::new().unwrap(); + let iam_path = tmp.path().join(".myfsio.sys").join("config"); + std::fs::create_dir_all(&iam_path).unwrap(); + + let iam_json = serde_json::json!({ + "version": 2, + "users": [{ + "user_id": "u-test1234", + "display_name": "admin", + "enabled": true, + "access_keys": [{ + "access_key": TEST_ACCESS_KEY, + "secret_key": TEST_SECRET_KEY, + "status": "active" + }], + "policies": [{ + "bucket": "*", + "actions": ["*"], + "prefix": "*" + }] + }] + }); + std::fs::write(iam_path.join("iam.json"), iam_json.to_string()).unwrap(); + + let config = myfsio_server::config::ServerConfig { + bind_addr: "127.0.0.1:0".parse().unwrap(), + storage_root: tmp.path().to_path_buf(), + region: "us-east-1".to_string(), + iam_config_path: iam_path.join("iam.json"), + sigv4_timestamp_tolerance_secs: 900, + presigned_url_min_expiry: 1, + presigned_url_max_expiry: 604800, + secret_key: None, + encryption_enabled: false, + kms_enabled: false, + gc_enabled: false, + integrity_enabled: false, + metrics_enabled: false, + lifecycle_enabled: false, + }; + let state = myfsio_server::state::AppState::new(config); + let app = myfsio_server::create_router(state); + (app, tmp) +} + +fn signed_request(method: Method, uri: &str, body: Body) -> Request { + Request::builder() + .method(method) + .uri(uri) + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(body) + .unwrap() +} + +#[tokio::test] +async fn test_unauthenticated_request_rejected() { + let (app, _tmp) = test_app(); + let resp = app + .oneshot(Request::builder().uri("/").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +async fn test_list_buckets_empty() { + let (app, _tmp) = test_app(); + let resp = app + .oneshot(signed_request(Method::GET, "/", Body::empty())) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("ListAllMyBucketsResult")); +} + +#[tokio::test] +async fn test_create_and_list_bucket() { + let (app, _tmp) = test_app(); + + let resp = app + .clone() + .oneshot(signed_request(Method::PUT, "/test-bucket", Body::empty())) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let resp = app + .oneshot(signed_request(Method::GET, "/", Body::empty())) + .await + .unwrap(); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("test-bucket")); +} + +#[tokio::test] +async fn test_head_bucket() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/my-bucket", Body::empty())) + .await + .unwrap(); + + let resp = app + .clone() + .oneshot(signed_request(Method::HEAD, "/my-bucket", Body::empty())) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers().get("x-amz-bucket-region").unwrap(), + "us-east-1" + ); + + let resp = app + .oneshot(signed_request(Method::HEAD, "/nonexistent", Body::empty())) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn test_delete_bucket() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/del-bucket", Body::empty())) + .await + .unwrap(); + + let resp = app + .clone() + .oneshot(signed_request(Method::DELETE, "/del-bucket", Body::empty())) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let resp = app + .oneshot(signed_request(Method::HEAD, "/del-bucket", Body::empty())) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn test_put_and_get_object() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/data-bucket", Body::empty())) + .await + .unwrap(); + + let resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/data-bucket/hello.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("content-type", "text/plain") + .body(Body::from("Hello, World!")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert!(resp.headers().get("etag").is_some()); + + let resp = app + .oneshot(signed_request( + Method::GET, + "/data-bucket/hello.txt", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(resp.headers().get("content-type").unwrap(), "text/plain"); + assert_eq!(resp.headers().get("content-length").unwrap(), "13"); + let body = resp.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(&body[..], b"Hello, World!"); +} + +#[tokio::test] +async fn test_head_object() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/hd-bucket", Body::empty())) + .await + .unwrap(); + + app.clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/hd-bucket/file.bin") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from(vec![0u8; 256])) + .unwrap(), + ) + .await + .unwrap(); + + let resp = app + .clone() + .oneshot(signed_request(Method::HEAD, "/hd-bucket/file.bin", Body::empty())) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(resp.headers().get("content-length").unwrap(), "256"); + assert!(resp.headers().get("etag").is_some()); + assert!(resp.headers().get("last-modified").is_some()); + + let resp = app + .oneshot(signed_request( + Method::HEAD, + "/hd-bucket/nonexistent.txt", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn test_delete_object() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/rm-bucket", Body::empty())) + .await + .unwrap(); + + app.clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/rm-bucket/removeme.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from("bye")) + .unwrap(), + ) + .await + .unwrap(); + + let resp = app + .clone() + .oneshot(signed_request( + Method::DELETE, + "/rm-bucket/removeme.txt", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let resp = app + .oneshot(signed_request( + Method::HEAD, + "/rm-bucket/removeme.txt", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn test_list_objects_v2() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/list-bucket", Body::empty())) + .await + .unwrap(); + + for name in ["a.txt", "b.txt", "dir/c.txt"] { + app.clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri(format!("/list-bucket/{}", name)) + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from("data")) + .unwrap(), + ) + .await + .unwrap(); + } + + let resp = app + .clone() + .oneshot(signed_request( + Method::GET, + "/list-bucket?list-type=2", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("a.txt")); + assert!(body.contains("b.txt")); + assert!(body.contains("dir/c.txt")); + assert!(body.contains("3")); + + let resp = app + .clone() + .oneshot(signed_request( + Method::GET, + "/list-bucket?list-type=2&delimiter=/", + Body::empty(), + )) + .await + .unwrap(); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("a.txt")); + assert!(body.contains("b.txt")); + assert!(body.contains("dir/")); + + let resp = app + .oneshot(signed_request( + Method::GET, + "/list-bucket?list-type=2&prefix=dir/", + Body::empty(), + )) + .await + .unwrap(); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("dir/c.txt")); + assert!(!body.contains("a.txt")); +} + +#[tokio::test] +async fn test_get_nonexistent_object_returns_404() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/err-bucket", Body::empty())) + .await + .unwrap(); + + let resp = app + .oneshot(signed_request( + Method::GET, + "/err-bucket/nope.txt", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("NoSuchKey")); +} + +#[tokio::test] +async fn test_create_duplicate_bucket_returns_409() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/dup-bucket", Body::empty())) + .await + .unwrap(); + + let resp = app + .oneshot(signed_request(Method::PUT, "/dup-bucket", Body::empty())) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::CONFLICT); +} + +#[tokio::test] +async fn test_delete_nonempty_bucket_returns_409() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/full-bucket", Body::empty())) + .await + .unwrap(); + + app.clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/full-bucket/obj.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from("data")) + .unwrap(), + ) + .await + .unwrap(); + + let resp = app + .oneshot(signed_request( + Method::DELETE, + "/full-bucket", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::CONFLICT); +} + +#[tokio::test] +async fn test_object_with_user_metadata() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/meta-bucket", Body::empty())) + .await + .unwrap(); + + app.clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/meta-bucket/tagged.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("x-amz-meta-author", "test-user") + .header("x-amz-meta-version", "42") + .body(Body::from("content")) + .unwrap(), + ) + .await + .unwrap(); + + let resp = app + .oneshot(signed_request( + Method::HEAD, + "/meta-bucket/tagged.txt", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers().get("x-amz-meta-author").unwrap(), + "test-user" + ); + assert_eq!(resp.headers().get("x-amz-meta-version").unwrap(), "42"); +} + +#[tokio::test] +async fn test_wrong_credentials_rejected() { + let (app, _tmp) = test_app(); + let resp = app + .oneshot( + Request::builder() + .uri("/") + .header("x-access-key", "WRONGKEY") + .header("x-secret-key", "WRONGSECRET") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +async fn test_server_header_present() { + let (app, _tmp) = test_app(); + let resp = app + .oneshot(signed_request(Method::GET, "/", Body::empty())) + .await + .unwrap(); + let server = resp.headers().get("server").unwrap().to_str().unwrap(); + assert!(server.starts_with("MyFSIO-Rust/")); +} + +#[tokio::test] +async fn test_range_request() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/range-bucket", Body::empty())) + .await + .unwrap(); + + let data = "Hello, World! This is range test data."; + app.clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/range-bucket/range.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from(data)) + .unwrap(), + ) + .await + .unwrap(); + + let resp = app + .clone() + .oneshot( + Request::builder() + .uri("/range-bucket/range.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("range", "bytes=0-4") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(resp.headers().get("content-length").unwrap(), "5"); + assert!(resp.headers().get("content-range").unwrap().to_str().unwrap().starts_with("bytes 0-4/")); + let body = resp.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(&body[..], b"Hello"); + + let resp = app + .oneshot( + Request::builder() + .uri("/range-bucket/range.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("range", "bytes=-5") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::PARTIAL_CONTENT); + let body = resp.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(&body[..], b"data."); +} + +#[tokio::test] +async fn test_copy_object() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/src-bucket", Body::empty())) + .await + .unwrap(); + app.clone() + .oneshot(signed_request(Method::PUT, "/dst-bucket", Body::empty())) + .await + .unwrap(); + + app.clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/src-bucket/original.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from("copy me")) + .unwrap(), + ) + .await + .unwrap(); + + let resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/dst-bucket/copied.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("x-amz-copy-source", "/src-bucket/original.txt") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("CopyObjectResult")); + + let resp = app + .oneshot(signed_request( + Method::GET, + "/dst-bucket/copied.txt", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = resp.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(&body[..], b"copy me"); +} + +#[tokio::test] +async fn test_multipart_upload_http() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/mp-bucket", Body::empty())) + .await + .unwrap(); + + let resp = app + .clone() + .oneshot(signed_request( + Method::POST, + "/mp-bucket/big-file.bin?uploads", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("InitiateMultipartUploadResult")); + assert!(body.contains("big-file.bin")); + + let upload_id = body + .split("") + .nth(1) + .unwrap() + .split("") + .next() + .unwrap(); + + let part1_data = vec![b'A'; 1024]; + let resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri(format!( + "/mp-bucket/big-file.bin?uploadId={}&partNumber=1", + upload_id + )) + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from(part1_data)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let etag1 = resp.headers().get("etag").unwrap().to_str().unwrap().trim_matches('"').to_string(); + + let part2_data = vec![b'B'; 512]; + let resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri(format!( + "/mp-bucket/big-file.bin?uploadId={}&partNumber=2", + upload_id + )) + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from(part2_data)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let etag2 = resp.headers().get("etag").unwrap().to_str().unwrap().trim_matches('"').to_string(); + + let complete_xml = format!( + "1\"{etag1}\"2\"{etag2}\"" + ); + + let resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::POST) + .uri(format!( + "/mp-bucket/big-file.bin?uploadId={}", + upload_id + )) + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from(complete_xml)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("CompleteMultipartUploadResult")); + + let resp = app + .oneshot(signed_request( + Method::HEAD, + "/mp-bucket/big-file.bin", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(resp.headers().get("content-length").unwrap(), "1536"); +} + +#[tokio::test] +async fn test_delete_objects_batch() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/batch-bucket", Body::empty())) + .await + .unwrap(); + + for name in ["a.txt", "b.txt", "c.txt"] { + app.clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri(format!("/batch-bucket/{}", name)) + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from("data")) + .unwrap(), + ) + .await + .unwrap(); + } + + let delete_xml = r#"a.txtb.txt"#; + + let resp = app + .clone() + .oneshot( + Request::builder() + .method(Method::POST) + .uri("/batch-bucket?delete") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from(delete_xml)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("DeleteResult")); + assert!(body.contains("a.txt")); + assert!(body.contains("b.txt")); + + let resp = app + .clone() + .oneshot(signed_request( + Method::HEAD, + "/batch-bucket/a.txt", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + + let resp = app + .oneshot(signed_request( + Method::HEAD, + "/batch-bucket/c.txt", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); +} + +#[tokio::test] +async fn test_bucket_versioning() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/ver-bucket", Body::empty())) + .await + .unwrap(); + + let resp = app + .clone() + .oneshot(signed_request( + Method::GET, + "/ver-bucket?versioning", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("VersioningConfiguration")); + assert!(body.contains("Suspended")); + + app.clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/ver-bucket?versioning") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from( + "Enabled", + )) + .unwrap(), + ) + .await + .unwrap(); + + let resp = app + .oneshot(signed_request( + Method::GET, + "/ver-bucket?versioning", + Body::empty(), + )) + .await + .unwrap(); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("Enabled")); +} + +#[tokio::test] +async fn test_bucket_tagging() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/tag-bucket", Body::empty())) + .await + .unwrap(); + + app.clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/tag-bucket?tagging") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from( + "envprod", + )) + .unwrap(), + ) + .await + .unwrap(); + + let resp = app + .clone() + .oneshot(signed_request( + Method::GET, + "/tag-bucket?tagging", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("env")); + assert!(body.contains("prod")); + + let resp = app + .oneshot(signed_request( + Method::DELETE, + "/tag-bucket?tagging", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); +} + +#[tokio::test] +async fn test_bucket_location() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/loc-bucket", Body::empty())) + .await + .unwrap(); + + let resp = app + .oneshot(signed_request( + Method::GET, + "/loc-bucket?location", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("LocationConstraint")); + assert!(body.contains("us-east-1")); +} + +#[tokio::test] +async fn test_bucket_cors() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/cors-bucket", Body::empty())) + .await + .unwrap(); + + let resp = app + .clone() + .oneshot(signed_request( + Method::GET, + "/cors-bucket?cors", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + + app.clone() + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/cors-bucket?cors") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from("*")) + .unwrap(), + ) + .await + .unwrap(); + + let resp = app + .clone() + .oneshot(signed_request( + Method::GET, + "/cors-bucket?cors", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let resp = app + .oneshot(signed_request( + Method::DELETE, + "/cors-bucket?cors", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); +} + +#[tokio::test] +async fn test_bucket_acl() { + let (app, _tmp) = test_app(); + + app.clone() + .oneshot(signed_request(Method::PUT, "/acl-bucket", Body::empty())) + .await + .unwrap(); + + let resp = app + .oneshot(signed_request( + Method::GET, + "/acl-bucket?acl", + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("AccessControlPolicy")); + assert!(body.contains("FULL_CONTROL")); +} + +#[tokio::test] +async fn test_object_tagging() { + let (app, _tmp) = test_app(); + let app = app.into_service(); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request(Method::PUT, "/tag-bucket", Body::empty()), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request( + Method::PUT, + "/tag-bucket/myfile.txt", + Body::from("file content"), + ), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request(Method::GET, "/tag-bucket/myfile.txt?tagging", Body::empty()), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("")); + assert!(body.contains("")); + + let tag_xml = r#"envprod"#; + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request( + Method::PUT, + "/tag-bucket/myfile.txt?tagging", + Body::from(tag_xml), + ), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request(Method::GET, "/tag-bucket/myfile.txt?tagging", Body::empty()), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("env")); + assert!(body.contains("prod")); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request(Method::DELETE, "/tag-bucket/myfile.txt?tagging", Body::empty()), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request(Method::GET, "/tag-bucket/myfile.txt?tagging", Body::empty()), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(!body.contains("env")); +} + +#[tokio::test] +async fn test_object_acl() { + let (app, _tmp) = test_app(); + let app = app.into_service(); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request(Method::PUT, "/acl-obj-bucket", Body::empty()), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request( + Method::PUT, + "/acl-obj-bucket/myfile.txt", + Body::from("content"), + ), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request(Method::GET, "/acl-obj-bucket/myfile.txt?acl", Body::empty()), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("AccessControlPolicy")); + assert!(body.contains("FULL_CONTROL")); +} + +#[tokio::test] +async fn test_object_legal_hold() { + let (app, _tmp) = test_app(); + let app = app.into_service(); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request(Method::PUT, "/lh-bucket", Body::empty()), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request( + Method::PUT, + "/lh-bucket/obj.txt", + Body::from("data"), + ), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request(Method::GET, "/lh-bucket/obj.txt?legal-hold", Body::empty()), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = String::from_utf8( + resp.into_body().collect().await.unwrap().to_bytes().to_vec(), + ) + .unwrap(); + assert!(body.contains("OFF")); +} + +async fn test_app_encrypted() -> (axum::Router, tempfile::TempDir) { + let tmp = tempfile::TempDir::new().unwrap(); + let iam_path = tmp.path().join(".myfsio.sys").join("config"); + std::fs::create_dir_all(&iam_path).unwrap(); + + let iam_json = serde_json::json!({ + "version": 2, + "users": [{ + "user_id": "u-test1234", + "display_name": "admin", + "enabled": true, + "access_keys": [{ + "access_key": TEST_ACCESS_KEY, + "secret_key": TEST_SECRET_KEY, + "status": "active" + }], + "policies": [{ + "bucket": "*", + "actions": ["*"], + "prefix": "*" + }] + }] + }); + std::fs::write(iam_path.join("iam.json"), iam_json.to_string()).unwrap(); + + let config = myfsio_server::config::ServerConfig { + bind_addr: "127.0.0.1:0".parse().unwrap(), + storage_root: tmp.path().to_path_buf(), + region: "us-east-1".to_string(), + iam_config_path: iam_path.join("iam.json"), + sigv4_timestamp_tolerance_secs: 900, + presigned_url_min_expiry: 1, + presigned_url_max_expiry: 604800, + secret_key: None, + encryption_enabled: true, + kms_enabled: true, + gc_enabled: false, + integrity_enabled: false, + metrics_enabled: false, + lifecycle_enabled: false, + }; + let state = myfsio_server::state::AppState::new_with_encryption(config).await; + let app = myfsio_server::create_router(state); + (app, tmp) +} + +#[tokio::test] +async fn test_sse_s3_encrypt_decrypt_roundtrip() { + let (app, _tmp) = test_app_encrypted().await; + let app = app.into_service(); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request(Method::PUT, "/enc-bucket", Body::empty()), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let plaintext = "This is secret data that should be encrypted at rest!"; + let req = Request::builder() + .method(Method::PUT) + .uri("/enc-bucket/secret.txt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("x-amz-server-side-encryption", "AES256") + .header("content-type", "text/plain") + .body(Body::from(plaintext)) + .unwrap(); + + let resp = tower::ServiceExt::oneshot(app.clone(), req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers().get("x-amz-server-side-encryption").unwrap(), + "AES256" + ); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request(Method::GET, "/enc-bucket/secret.txt", Body::empty()), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers().get("x-amz-server-side-encryption").unwrap(), + "AES256" + ); + let body = resp + .into_body() + .collect() + .await + .unwrap() + .to_bytes() + .to_vec(); + assert_eq!(String::from_utf8(body).unwrap(), plaintext); +} + +#[tokio::test] +async fn test_kms_key_crud() { + let (app, _tmp) = test_app_encrypted().await; + let app = app.into_service(); + + let req = Request::builder() + .method(Method::POST) + .uri("/kms/keys") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .header("content-type", "application/json") + .body(Body::from(r#"{"Description": "test key"}"#)) + .unwrap(); + let resp = tower::ServiceExt::oneshot(app.clone(), req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body: serde_json::Value = serde_json::from_slice( + &resp.into_body().collect().await.unwrap().to_bytes(), + ) + .unwrap(); + let key_id = body["KeyId"].as_str().unwrap().to_string(); + assert!(!key_id.is_empty()); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request(Method::GET, "/kms/keys", Body::empty()), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body: serde_json::Value = serde_json::from_slice( + &resp.into_body().collect().await.unwrap().to_bytes(), + ) + .unwrap(); + assert_eq!(body["keys"].as_array().unwrap().len(), 1); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request( + Method::GET, + &format!("/kms/keys/{}", key_id), + Body::empty(), + ), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let resp = tower::ServiceExt::oneshot( + app.clone(), + signed_request( + Method::DELETE, + &format!("/kms/keys/{}", key_id), + Body::empty(), + ), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); +} + +#[tokio::test] +async fn test_kms_encrypt_decrypt() { + let (app, _tmp) = test_app_encrypted().await; + let app = app.into_service(); + + let req = Request::builder() + .method(Method::POST) + .uri("/kms/keys") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from(r#"{"Description": "enc key"}"#)) + .unwrap(); + let resp = tower::ServiceExt::oneshot(app.clone(), req).await.unwrap(); + let body: serde_json::Value = serde_json::from_slice( + &resp.into_body().collect().await.unwrap().to_bytes(), + ) + .unwrap(); + let key_id = body["KeyId"].as_str().unwrap().to_string(); + + use base64::engine::general_purpose::STANDARD as B64; + use base64::Engine; + + let plaintext = b"Hello KMS!"; + let enc_req = serde_json::json!({ + "KeyId": key_id, + "Plaintext": B64.encode(plaintext), + }); + let req = Request::builder() + .method(Method::POST) + .uri("/kms/encrypt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from(enc_req.to_string())) + .unwrap(); + let resp = tower::ServiceExt::oneshot(app.clone(), req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body: serde_json::Value = serde_json::from_slice( + &resp.into_body().collect().await.unwrap().to_bytes(), + ) + .unwrap(); + let ct_b64 = body["CiphertextBlob"].as_str().unwrap().to_string(); + + let dec_req = serde_json::json!({ + "KeyId": key_id, + "CiphertextBlob": ct_b64, + }); + let req = Request::builder() + .method(Method::POST) + .uri("/kms/decrypt") + .header("x-access-key", TEST_ACCESS_KEY) + .header("x-secret-key", TEST_SECRET_KEY) + .body(Body::from(dec_req.to_string())) + .unwrap(); + let resp = tower::ServiceExt::oneshot(app.clone(), req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body: serde_json::Value = serde_json::from_slice( + &resp.into_body().collect().await.unwrap().to_bytes(), + ) + .unwrap(); + let pt_b64 = body["Plaintext"].as_str().unwrap(); + let result = B64.decode(pt_b64).unwrap(); + assert_eq!(result, plaintext); +} + diff --git a/myfsio-engine/crates/myfsio-storage/Cargo.toml b/myfsio-engine/crates/myfsio-storage/Cargo.toml new file mode 100644 index 0000000..454b11b --- /dev/null +++ b/myfsio-engine/crates/myfsio-storage/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "myfsio-storage" +version = "0.1.0" +edition = "2021" + +[dependencies] +myfsio-common = { path = "../myfsio-common" } +myfsio-crypto = { path = "../myfsio-crypto" } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +dashmap = { workspace = true } +parking_lot = { workspace = true } +uuid = { workspace = true } +chrono = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +regex = { workspace = true } +unicode-normalization = { workspace = true } +md-5 = { workspace = true } +sha2 = { workspace = true } +hex = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tempfile = "3" diff --git a/myfsio-engine/crates/myfsio-storage/src/error.rs b/myfsio-engine/crates/myfsio-storage/src/error.rs new file mode 100644 index 0000000..5b26cc2 --- /dev/null +++ b/myfsio-engine/crates/myfsio-storage/src/error.rs @@ -0,0 +1,59 @@ +use myfsio_common::error::{S3Error, S3ErrorCode}; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum StorageError { + #[error("Bucket not found: {0}")] + BucketNotFound(String), + #[error("Bucket already exists: {0}")] + BucketAlreadyExists(String), + #[error("Bucket not empty: {0}")] + BucketNotEmpty(String), + #[error("Object not found: {bucket}/{key}")] + ObjectNotFound { bucket: String, key: String }, + #[error("Invalid bucket name: {0}")] + InvalidBucketName(String), + #[error("Invalid object key: {0}")] + InvalidObjectKey(String), + #[error("Upload not found: {0}")] + UploadNotFound(String), + #[error("Quota exceeded: {0}")] + QuotaExceeded(String), + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + #[error("JSON error: {0}")] + Json(#[from] serde_json::Error), + #[error("Internal error: {0}")] + Internal(String), +} + +impl From for S3Error { + fn from(err: StorageError) -> Self { + match err { + StorageError::BucketNotFound(name) => { + S3Error::from_code(S3ErrorCode::NoSuchBucket).with_resource(format!("/{}", name)) + } + StorageError::BucketAlreadyExists(name) => { + S3Error::from_code(S3ErrorCode::BucketAlreadyExists) + .with_resource(format!("/{}", name)) + } + StorageError::BucketNotEmpty(name) => { + S3Error::from_code(S3ErrorCode::BucketNotEmpty) + .with_resource(format!("/{}", name)) + } + StorageError::ObjectNotFound { bucket, key } => { + S3Error::from_code(S3ErrorCode::NoSuchKey) + .with_resource(format!("/{}/{}", bucket, key)) + } + StorageError::InvalidBucketName(msg) => S3Error::new(S3ErrorCode::InvalidBucketName, msg), + StorageError::InvalidObjectKey(msg) => S3Error::new(S3ErrorCode::InvalidKey, msg), + StorageError::UploadNotFound(id) => { + S3Error::new(S3ErrorCode::NoSuchUpload, format!("Upload {} not found", id)) + } + StorageError::QuotaExceeded(msg) => S3Error::new(S3ErrorCode::QuotaExceeded, msg), + StorageError::Io(e) => S3Error::new(S3ErrorCode::InternalError, e.to_string()), + StorageError::Json(e) => S3Error::new(S3ErrorCode::InternalError, e.to_string()), + StorageError::Internal(msg) => S3Error::new(S3ErrorCode::InternalError, msg), + } + } +} diff --git a/myfsio-engine/crates/myfsio-storage/src/fs_backend.rs b/myfsio-engine/crates/myfsio-storage/src/fs_backend.rs new file mode 100644 index 0000000..e3c2b0b --- /dev/null +++ b/myfsio-engine/crates/myfsio-storage/src/fs_backend.rs @@ -0,0 +1,1933 @@ +use crate::error::StorageError; +use crate::traits::{AsyncReadStream, StorageResult}; +use crate::validation; +use myfsio_common::constants::*; +use myfsio_common::types::*; + +use chrono::{DateTime, TimeZone, Utc}; +use dashmap::DashMap; +use md5::{Digest, Md5}; +use parking_lot::Mutex; +use serde_json::Value; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Instant; +use tokio::io::AsyncReadExt; +use uuid::Uuid; + +pub struct FsStorageBackend { + root: PathBuf, + object_key_max_length_bytes: usize, + bucket_config_cache: DashMap, + bucket_config_cache_ttl: std::time::Duration, + meta_read_cache: DashMap<(String, String), Option>>, + meta_index_locks: DashMap>>, + stats_cache: DashMap, + stats_cache_ttl: std::time::Duration, +} + +impl FsStorageBackend { + pub fn new(root: PathBuf) -> Self { + let backend = Self { + root, + object_key_max_length_bytes: DEFAULT_OBJECT_KEY_MAX_BYTES, + bucket_config_cache: DashMap::new(), + bucket_config_cache_ttl: std::time::Duration::from_secs(30), + meta_read_cache: DashMap::new(), + meta_index_locks: DashMap::new(), + stats_cache: DashMap::new(), + stats_cache_ttl: std::time::Duration::from_secs(60), + }; + backend.ensure_system_roots(); + backend + } + + fn ensure_system_roots(&self) { + let dirs = [ + self.system_root_path(), + self.system_buckets_root(), + self.multipart_root(), + self.system_root_path().join("tmp"), + ]; + for dir in &dirs { + std::fs::create_dir_all(dir).ok(); + } + } + + fn bucket_path(&self, bucket_name: &str) -> PathBuf { + self.root.join(bucket_name) + } + + fn system_root_path(&self) -> PathBuf { + self.root.join(SYSTEM_ROOT) + } + + fn system_buckets_root(&self) -> PathBuf { + self.system_root_path().join(SYSTEM_BUCKETS_DIR) + } + + fn system_bucket_root(&self, bucket_name: &str) -> PathBuf { + self.system_buckets_root().join(bucket_name) + } + + fn bucket_meta_root(&self, bucket_name: &str) -> PathBuf { + self.system_bucket_root(bucket_name).join(BUCKET_META_DIR) + } + + fn bucket_versions_root(&self, bucket_name: &str) -> PathBuf { + self.system_bucket_root(bucket_name).join(BUCKET_VERSIONS_DIR) + } + + fn multipart_root(&self) -> PathBuf { + self.system_root_path().join(SYSTEM_MULTIPART_DIR) + } + + fn multipart_bucket_root(&self, bucket_name: &str) -> PathBuf { + self.multipart_root().join(bucket_name) + } + + fn tmp_dir(&self) -> PathBuf { + self.system_root_path().join("tmp") + } + + fn object_path(&self, bucket_name: &str, object_key: &str) -> StorageResult { + self.validate_key(object_key)?; + Ok(self.bucket_path(bucket_name).join(object_key)) + } + + fn validate_key(&self, object_key: &str) -> StorageResult<()> { + let is_windows = cfg!(windows); + if let Some(err) = validation::validate_object_key( + object_key, + self.object_key_max_length_bytes, + is_windows, + None, + ) { + return Err(StorageError::InvalidObjectKey(err)); + } + Ok(()) + } + + fn require_bucket(&self, bucket_name: &str) -> StorageResult { + let path = self.bucket_path(bucket_name); + if !path.exists() { + return Err(StorageError::BucketNotFound(bucket_name.to_string())); + } + Ok(path) + } + + fn index_file_for_key(&self, bucket_name: &str, key: &str) -> (PathBuf, String) { + let meta_root = self.bucket_meta_root(bucket_name); + let key_path = Path::new(key); + let entry_name = key_path + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| key.to_string()); + + let parent = key_path.parent(); + match parent { + Some(p) if p != Path::new("") && p != Path::new(".") => { + (meta_root.join(p).join(INDEX_FILE), entry_name) + } + _ => (meta_root.join(INDEX_FILE), entry_name), + } + } + + fn get_meta_index_lock(&self, index_path: &str) -> Arc> { + self.meta_index_locks + .entry(index_path.to_string()) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone() + } + + fn bucket_config_path(&self, bucket_name: &str) -> PathBuf { + self.system_bucket_root(bucket_name).join(BUCKET_CONFIG_FILE) + } + + fn version_dir(&self, bucket_name: &str, key: &str) -> PathBuf { + self.bucket_versions_root(bucket_name).join(key) + } + + fn legacy_meta_root(&self, bucket_name: &str) -> PathBuf { + self.bucket_path(bucket_name).join(".meta") + } + + fn legacy_metadata_file(&self, bucket_name: &str, key: &str) -> PathBuf { + self.legacy_meta_root(bucket_name) + .join(format!("{}.meta.json", key)) + } + + fn legacy_versions_root(&self, bucket_name: &str) -> PathBuf { + self.bucket_path(bucket_name).join(".versions") + } + + fn legacy_multipart_root(&self, bucket_name: &str) -> PathBuf { + self.bucket_path(bucket_name).join(".multipart") + } +} + +impl FsStorageBackend { + fn atomic_write_json_sync(path: &Path, data: &Value, sync: bool) -> std::io::Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let tmp_path = path.with_extension("tmp"); + let result = (|| { + let file = std::fs::File::create(&tmp_path)?; + let mut writer = std::io::BufWriter::new(file); + serde_json::to_writer(&mut writer, data) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + let file = writer.into_inner()?; + if sync { + file.sync_all()?; + } + drop(file); + std::fs::rename(&tmp_path, path)?; + Ok(()) + })(); + if result.is_err() { + let _ = std::fs::remove_file(&tmp_path); + } + result + } + + fn read_index_entry_sync( + &self, + bucket_name: &str, + key: &str, + ) -> Option> { + let cache_key = (bucket_name.to_string(), key.to_string()); + if let Some(entry) = self.meta_read_cache.get(&cache_key) { + return entry.value().clone(); + } + + let (index_path, entry_name) = self.index_file_for_key(bucket_name, key); + let result = if index_path.exists() { + std::fs::read_to_string(&index_path) + .ok() + .and_then(|s| serde_json::from_str::>(&s).ok()) + .and_then(|index| { + index.get(&entry_name).and_then(|v| { + if let Value::Object(map) = v { + Some( + map.iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + ) + } else { + None + } + }) + }) + } else { + None + }; + + self.meta_read_cache.insert(cache_key, result.clone()); + result + } + + fn write_index_entry_sync( + &self, + bucket_name: &str, + key: &str, + entry: &HashMap, + ) -> std::io::Result<()> { + let (index_path, entry_name) = self.index_file_for_key(bucket_name, key); + let lock = self.get_meta_index_lock(&index_path.to_string_lossy()); + let _guard = lock.lock(); + + if let Some(parent) = index_path.parent() { + std::fs::create_dir_all(parent)?; + } + + let mut index_data: HashMap = if index_path.exists() { + std::fs::read_to_string(&index_path) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default() + } else { + HashMap::new() + }; + + index_data.insert( + entry_name, + serde_json::to_value(entry) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?, + ); + + let json_val = serde_json::to_value(&index_data) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + Self::atomic_write_json_sync(&index_path, &json_val, true)?; + + let cache_key = (bucket_name.to_string(), key.to_string()); + self.meta_read_cache.remove(&cache_key); + + Ok(()) + } + + fn delete_index_entry_sync(&self, bucket_name: &str, key: &str) -> std::io::Result<()> { + let (index_path, entry_name) = self.index_file_for_key(bucket_name, key); + if !index_path.exists() { + let cache_key = (bucket_name.to_string(), key.to_string()); + self.meta_read_cache.remove(&cache_key); + return Ok(()); + } + + let lock = self.get_meta_index_lock(&index_path.to_string_lossy()); + let _guard = lock.lock(); + + let mut index_data: HashMap = std::fs::read_to_string(&index_path) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default(); + + if index_data.remove(&entry_name).is_some() { + if index_data.is_empty() { + let _ = std::fs::remove_file(&index_path); + } else { + let json_val = serde_json::to_value(&index_data) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + Self::atomic_write_json_sync(&index_path, &json_val, true)?; + } + } + + let cache_key = (bucket_name.to_string(), key.to_string()); + self.meta_read_cache.remove(&cache_key); + Ok(()) + } + + fn read_metadata_sync(&self, bucket_name: &str, key: &str) -> HashMap { + if let Some(entry) = self.read_index_entry_sync(bucket_name, key) { + if let Some(Value::Object(meta)) = entry.get("metadata") { + return meta + .iter() + .filter_map(|(k, v)| v.as_str().map(|s| (k.clone(), s.to_string()))) + .collect(); + } + } + + for meta_file in [ + self.bucket_meta_root(bucket_name) + .join(format!("{}.meta.json", key)), + self.legacy_metadata_file(bucket_name, key), + ] { + if meta_file.exists() { + if let Ok(content) = std::fs::read_to_string(&meta_file) { + if let Ok(payload) = serde_json::from_str::(&content) { + if let Some(Value::Object(meta)) = payload.get("metadata") { + return meta + .iter() + .filter_map(|(k, v)| { + v.as_str().map(|s| (k.clone(), s.to_string())) + }) + .collect(); + } + } + } + } + } + + HashMap::new() + } + + fn write_metadata_sync( + &self, + bucket_name: &str, + key: &str, + metadata: &HashMap, + ) -> std::io::Result<()> { + if metadata.is_empty() { + return self.delete_index_entry_sync(bucket_name, key); + } + + let mut entry = HashMap::new(); + let meta_value = serde_json::to_value(metadata) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + entry.insert("metadata".to_string(), meta_value); + self.write_index_entry_sync(bucket_name, key, &entry)?; + + let old_meta = self + .bucket_meta_root(bucket_name) + .join(format!("{}.meta.json", key)); + if old_meta.exists() { + let _ = std::fs::remove_file(&old_meta); + } + + Ok(()) + } + + fn delete_metadata_sync(&self, bucket_name: &str, key: &str) -> std::io::Result<()> { + self.delete_index_entry_sync(bucket_name, key)?; + + for meta_file in [ + self.bucket_meta_root(bucket_name) + .join(format!("{}.meta.json", key)), + self.legacy_metadata_file(bucket_name, key), + ] { + if meta_file.exists() { + let _ = std::fs::remove_file(&meta_file); + } + } + + Ok(()) + } + + fn compute_etag_sync(path: &Path) -> std::io::Result { + myfsio_crypto::hashing::md5_file(path) + } + + fn read_bucket_config_sync(&self, bucket_name: &str) -> BucketConfig { + if let Some(entry) = self.bucket_config_cache.get(bucket_name) { + let (config, cached_at) = entry.value(); + if cached_at.elapsed() < self.bucket_config_cache_ttl { + return config.clone(); + } + } + + let config_path = self.bucket_config_path(bucket_name); + let config = if config_path.exists() { + std::fs::read_to_string(&config_path) + .ok() + .and_then(|s| serde_json::from_str::(&s).ok()) + .unwrap_or_default() + } else { + BucketConfig::default() + }; + + self.bucket_config_cache + .insert(bucket_name.to_string(), (config.clone(), Instant::now())); + config + } + + fn write_bucket_config_sync( + &self, + bucket_name: &str, + config: &BucketConfig, + ) -> std::io::Result<()> { + let config_path = self.bucket_config_path(bucket_name); + let json_val = serde_json::to_value(config) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + Self::atomic_write_json_sync(&config_path, &json_val, true)?; + self.bucket_config_cache + .insert(bucket_name.to_string(), (config.clone(), Instant::now())); + Ok(()) + } + + fn check_bucket_contents_sync(&self, bucket_path: &Path) -> (bool, bool, bool) { + let bucket_name = bucket_path + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default(); + + let has_objects = Self::dir_has_files(bucket_path, Some(INTERNAL_FOLDERS)); + let has_versions = Self::dir_has_files(&self.bucket_versions_root(&bucket_name), None) + || Self::dir_has_files(&self.legacy_versions_root(&bucket_name), None); + let has_multipart = + Self::dir_has_files(&self.multipart_bucket_root(&bucket_name), None) + || Self::dir_has_files(&self.legacy_multipart_root(&bucket_name), None); + + (has_objects, has_versions, has_multipart) + } + + fn dir_has_files(dir: &Path, skip_dirs: Option<&[&str]>) -> bool { + if !dir.exists() { + return false; + } + let mut stack = vec![dir.to_path_buf()]; + while let Some(current) = stack.pop() { + let entries = match std::fs::read_dir(¤t) { + Ok(e) => e, + Err(_) => continue, + }; + for entry in entries.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if current == dir { + if let Some(skip) = skip_dirs { + if skip.contains(&name_str.as_ref()) { + continue; + } + } + } + let ft = match entry.file_type() { + Ok(ft) => ft, + Err(_) => continue, + }; + if ft.is_file() { + return true; + } + if ft.is_dir() { + stack.push(entry.path()); + } + } + } + false + } + + fn remove_tree(path: &Path) { + if path.exists() { + let _ = std::fs::remove_dir_all(path); + } + } + + fn safe_unlink(path: &Path) -> std::io::Result<()> { + for attempt in 0..3 { + match std::fs::remove_file(path) { + Ok(()) => return Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied && cfg!(windows) => { + if attempt < 2 { + std::thread::sleep(std::time::Duration::from_millis( + 150 * (attempt as u64 + 1), + )); + } else { + return Err(e); + } + } + Err(e) => return Err(e), + } + } + Ok(()) + } + + fn cleanup_empty_parents(path: &Path, stop_at: &Path) { + let mut parent = path.parent(); + while let Some(p) = parent { + if p == stop_at || !p.exists() { + break; + } + match std::fs::read_dir(p) { + Ok(mut entries) => { + if entries.next().is_some() { + break; + } + let _ = std::fs::remove_dir(p); + } + Err(_) => break, + } + parent = p.parent(); + } + } + + fn archive_current_version_sync( + &self, + bucket_name: &str, + key: &str, + reason: &str, + ) -> std::io::Result { + let bucket_path = self.bucket_path(bucket_name); + let source = bucket_path.join(key); + if !source.exists() { + return Ok(0); + } + + let version_dir = self.version_dir(bucket_name, key); + std::fs::create_dir_all(&version_dir)?; + + let now = Utc::now(); + let version_id = format!( + "{}-{}", + now.format("%Y%m%dT%H%M%S%6fZ"), + &Uuid::new_v4().to_string()[..8] + ); + + let data_path = version_dir.join(format!("{}.bin", version_id)); + std::fs::copy(&source, &data_path)?; + + let source_meta = source.metadata()?; + let source_size = source_meta.len(); + + let metadata = self.read_metadata_sync(bucket_name, key); + let etag = Self::compute_etag_sync(&source).unwrap_or_default(); + + let record = serde_json::json!({ + "version_id": version_id, + "key": key, + "size": source_size, + "archived_at": now.to_rfc3339(), + "etag": etag, + "metadata": metadata, + "reason": reason, + }); + + let manifest_path = version_dir.join(format!("{}.json", version_id)); + Self::atomic_write_json_sync(&manifest_path, &record, true)?; + + Ok(source_size) + } + + fn bucket_stats_sync(&self, bucket_name: &str) -> StorageResult { + let bucket_path = self.require_bucket(bucket_name)?; + + if let Some(entry) = self.stats_cache.get(bucket_name) { + let (stats, cached_at) = entry.value(); + if cached_at.elapsed() < self.stats_cache_ttl { + return Ok(stats.clone()); + } + } + + let mut object_count: u64 = 0; + let mut total_bytes: u64 = 0; + let mut version_count: u64 = 0; + let mut version_bytes: u64 = 0; + + let internal = INTERNAL_FOLDERS; + let bucket_str = bucket_path.to_string_lossy().to_string(); + let mut stack = vec![bucket_str.clone()]; + while let Some(current) = stack.pop() { + let entries = match std::fs::read_dir(¤t) { + Ok(e) => e, + Err(_) => continue, + }; + for entry in entries.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if current == bucket_str && internal.contains(&name_str.as_ref()) { + continue; + } + let ft = match entry.file_type() { + Ok(ft) => ft, + Err(_) => continue, + }; + if ft.is_dir() { + stack.push(entry.path().to_string_lossy().to_string()); + } else if ft.is_file() { + object_count += 1; + if let Ok(meta) = entry.metadata() { + total_bytes += meta.len(); + } + } + } + } + + let versions_root = self.bucket_versions_root(bucket_name); + if versions_root.exists() { + let mut v_stack = vec![versions_root.to_string_lossy().to_string()]; + while let Some(current) = v_stack.pop() { + let entries = match std::fs::read_dir(¤t) { + Ok(e) => e, + Err(_) => continue, + }; + for entry in entries.flatten() { + let ft = match entry.file_type() { + Ok(ft) => ft, + Err(_) => continue, + }; + if ft.is_dir() { + v_stack.push(entry.path().to_string_lossy().to_string()); + } else if ft.is_file() { + let name = entry.file_name(); + if name.to_string_lossy().ends_with(".bin") { + version_count += 1; + if let Ok(meta) = entry.metadata() { + version_bytes += meta.len(); + } + } + } + } + } + } + + let stats = BucketStats { + objects: object_count, + bytes: total_bytes, + version_count, + version_bytes, + }; + + self.stats_cache + .insert(bucket_name.to_string(), (stats.clone(), Instant::now())); + Ok(stats) + } + + fn list_objects_sync( + &self, + bucket_name: &str, + params: &ListParams, + ) -> StorageResult { + let bucket_path = self.require_bucket(bucket_name)?; + + let mut all_keys: Vec<(String, u64, f64, Option)> = Vec::new(); + let internal = INTERNAL_FOLDERS; + let bucket_str = bucket_path.to_string_lossy().to_string(); + let bucket_prefix_len = bucket_str.len() + 1; + let mut stack = vec![bucket_str.clone()]; + + while let Some(current) = stack.pop() { + let entries = match std::fs::read_dir(¤t) { + Ok(e) => e, + Err(_) => continue, + }; + for entry in entries.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if current == bucket_str && internal.contains(&name_str.as_ref()) { + continue; + } + let ft = match entry.file_type() { + Ok(ft) => ft, + Err(_) => continue, + }; + if ft.is_dir() { + stack.push(entry.path().to_string_lossy().to_string()); + } else if ft.is_file() { + let full_path = entry.path().to_string_lossy().to_string(); + let key = full_path[bucket_prefix_len..].replace('\\', "/"); + if let Ok(meta) = entry.metadata() { + let mtime = meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0); + all_keys.push((key, meta.len(), mtime, None)); + } + } + } + } + + all_keys.sort_by(|a, b| a.0.cmp(&b.0)); + + if let Some(ref prefix) = params.prefix { + all_keys.retain(|k| k.0.starts_with(prefix.as_str())); + } + + let start_idx = if let Some(ref token) = params.continuation_token { + all_keys + .iter() + .position(|k| k.0.as_str() > token.as_str()) + .unwrap_or(all_keys.len()) + } else if let Some(ref start_after) = params.start_after { + all_keys + .iter() + .position(|k| k.0.as_str() > start_after.as_str()) + .unwrap_or(all_keys.len()) + } else { + 0 + }; + + let max_keys = if params.max_keys == 0 { + DEFAULT_MAX_KEYS + } else { + params.max_keys + }; + + let end_idx = std::cmp::min(start_idx + max_keys, all_keys.len()); + let is_truncated = end_idx < all_keys.len(); + + let objects: Vec = all_keys[start_idx..end_idx] + .iter() + .map(|(key, size, mtime, etag)| { + let lm = Utc + .timestamp_opt(*mtime as i64, ((*mtime % 1.0) * 1_000_000_000.0) as u32) + .single() + .unwrap_or_else(Utc::now); + let mut obj = ObjectMeta::new(key.clone(), *size, lm); + obj.etag = etag.clone().or_else(|| { + let meta = self.read_metadata_sync(bucket_name, key); + meta.get("__etag__").cloned() + }); + obj + }) + .collect(); + + let next_token = if is_truncated { + objects.last().map(|o| o.key.clone()) + } else { + None + }; + + Ok(ListObjectsResult { + objects, + is_truncated, + next_continuation_token: next_token, + }) + } + + fn list_objects_shallow_sync( + &self, + bucket_name: &str, + params: &ShallowListParams, + ) -> StorageResult { + let bucket_path = self.require_bucket(bucket_name)?; + + let target_dir = if params.prefix.is_empty() { + bucket_path.clone() + } else { + let prefix_path = Path::new(¶ms.prefix); + let dir_part = if params.prefix.ends_with(¶ms.delimiter) { + prefix_path.to_path_buf() + } else { + prefix_path.parent().unwrap_or(Path::new("")).to_path_buf() + }; + bucket_path.join(dir_part) + }; + + if !target_dir.exists() { + return Ok(ShallowListResult { + objects: Vec::new(), + common_prefixes: Vec::new(), + is_truncated: false, + next_continuation_token: None, + }); + } + + let mut files = Vec::new(); + let mut dirs = Vec::new(); + + let entries = std::fs::read_dir(&target_dir).map_err(StorageError::Io)?; + for entry in entries.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy().to_string(); + + if target_dir == bucket_path && INTERNAL_FOLDERS.contains(&name_str.as_str()) { + continue; + } + + let ft = match entry.file_type() { + Ok(ft) => ft, + Err(_) => continue, + }; + + let rel = entry + .path() + .strip_prefix(&bucket_path) + .unwrap_or(Path::new("")) + .to_string_lossy() + .replace('\\', "/"); + + if !params.prefix.is_empty() && !rel.starts_with(¶ms.prefix) { + continue; + } + + if ft.is_dir() { + dirs.push(format!("{}{}", rel, ¶ms.delimiter)); + } else if ft.is_file() { + if let Ok(meta) = entry.metadata() { + let mtime = meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0); + let lm = Utc + .timestamp_opt(mtime as i64, ((mtime % 1.0) * 1_000_000_000.0) as u32) + .single() + .unwrap_or_else(Utc::now); + let etag = self + .read_metadata_sync(bucket_name, &rel) + .get("__etag__") + .cloned(); + let mut obj = ObjectMeta::new(rel, meta.len(), lm); + obj.etag = etag; + files.push(obj); + } + } + } + + files.sort_by(|a, b| a.key.cmp(&b.key)); + dirs.sort(); + + let mut merged: Vec = Vec::new(); + let mut fi = 0; + let mut di = 0; + while fi < files.len() && di < dirs.len() { + if files[fi].key < dirs[di] { + merged.push(Either::File(fi)); + fi += 1; + } else { + merged.push(Either::Dir(di)); + di += 1; + } + } + while fi < files.len() { + merged.push(Either::File(fi)); + fi += 1; + } + while di < dirs.len() { + merged.push(Either::Dir(di)); + di += 1; + } + + let start_idx = if let Some(ref token) = params.continuation_token { + merged + .iter() + .position(|e| match e { + Either::File(i) => files[*i].key.as_str() > token.as_str(), + Either::Dir(i) => dirs[*i].as_str() > token.as_str(), + }) + .unwrap_or(merged.len()) + } else { + 0 + }; + + let max_keys = if params.max_keys == 0 { + DEFAULT_MAX_KEYS + } else { + params.max_keys + }; + + let end_idx = std::cmp::min(start_idx + max_keys, merged.len()); + let is_truncated = end_idx < merged.len(); + + let mut result_objects = Vec::new(); + let mut result_prefixes = Vec::new(); + + for item in &merged[start_idx..end_idx] { + match item { + Either::File(i) => result_objects.push(files[*i].clone()), + Either::Dir(i) => result_prefixes.push(dirs[*i].clone()), + } + } + + let next_token = if is_truncated { + match &merged[end_idx - 1] { + Either::File(i) => Some(files[*i].key.clone()), + Either::Dir(i) => Some(dirs[*i].clone()), + } + } else { + None + }; + + Ok(ShallowListResult { + objects: result_objects, + common_prefixes: result_prefixes, + is_truncated, + next_continuation_token: next_token, + }) + } + + fn put_object_sync( + &self, + bucket_name: &str, + key: &str, + data: &[u8], + metadata: Option>, + ) -> StorageResult { + let bucket_path = self.require_bucket(bucket_name)?; + self.validate_key(key)?; + + let destination = bucket_path.join(key); + if let Some(parent) = destination.parent() { + std::fs::create_dir_all(parent).map_err(StorageError::Io)?; + } + + let is_overwrite = destination.exists(); + + let tmp_dir = self.tmp_dir(); + std::fs::create_dir_all(&tmp_dir).map_err(StorageError::Io)?; + let tmp_path = tmp_dir.join(format!("{}.tmp", Uuid::new_v4())); + + let mut hasher = Md5::new(); + hasher.update(data); + let etag = format!("{:x}", hasher.finalize()); + + std::fs::write(&tmp_path, data).map_err(StorageError::Io)?; + let new_size = data.len() as u64; + + let lock_dir = self.system_bucket_root(bucket_name).join("locks"); + std::fs::create_dir_all(&lock_dir).map_err(StorageError::Io)?; + + let versioning_enabled = self.read_bucket_config_sync(bucket_name).versioning_enabled; + if versioning_enabled && is_overwrite { + self.archive_current_version_sync(bucket_name, key, "overwrite") + .map_err(StorageError::Io)?; + } + + std::fs::rename(&tmp_path, &destination).map_err(|e| { + let _ = std::fs::remove_file(&tmp_path); + StorageError::Io(e) + })?; + + let file_meta = std::fs::metadata(&destination).map_err(StorageError::Io)?; + let mtime = file_meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0); + + let mut internal_meta = HashMap::new(); + internal_meta.insert("__etag__".to_string(), etag.clone()); + internal_meta.insert("__size__".to_string(), new_size.to_string()); + internal_meta.insert("__last_modified__".to_string(), mtime.to_string()); + + if let Some(ref user_meta) = metadata { + for (k, v) in user_meta { + internal_meta.insert(k.clone(), v.clone()); + } + } + + self.write_metadata_sync(bucket_name, key, &internal_meta) + .map_err(StorageError::Io)?; + + let lm = Utc + .timestamp_opt(mtime as i64, ((mtime % 1.0) * 1_000_000_000.0) as u32) + .single() + .unwrap_or_else(Utc::now); + + let mut obj = ObjectMeta::new(key.to_string(), new_size, lm); + obj.etag = Some(etag); + obj.metadata = metadata.unwrap_or_default(); + Ok(obj) + } +} + +enum Either { + File(usize), + Dir(usize), +} + +impl crate::traits::StorageEngine for FsStorageBackend { + async fn list_buckets(&self) -> StorageResult> { + let root = self.root.clone(); + tokio::task::spawn_blocking(move || { + let mut buckets = Vec::new(); + let entries = std::fs::read_dir(&root).map_err(StorageError::Io)?; + for entry in entries.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy().to_string(); + if name_str == SYSTEM_ROOT { + continue; + } + let ft = match entry.file_type() { + Ok(ft) => ft, + Err(_) => continue, + }; + if !ft.is_dir() { + continue; + } + let meta = match entry.metadata() { + Ok(m) => m, + Err(_) => continue, + }; + let created = meta + .created() + .or_else(|_| meta.modified()) + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| { + Utc.timestamp_opt(d.as_secs() as i64, d.subsec_nanos()) + .single() + .unwrap_or_else(Utc::now) + }) + .unwrap_or_else(Utc::now); + buckets.push(BucketMeta { + name: name_str, + creation_date: created, + }); + } + buckets.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(buckets) + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + } + + async fn create_bucket(&self, name: &str) -> StorageResult<()> { + if let Some(err) = validation::validate_bucket_name(name) { + return Err(StorageError::InvalidBucketName(err)); + } + let bucket_path = self.bucket_path(name); + if bucket_path.exists() { + return Err(StorageError::BucketAlreadyExists(name.to_string())); + } + std::fs::create_dir_all(&bucket_path).map_err(StorageError::Io)?; + std::fs::create_dir_all(self.system_bucket_root(name)).map_err(StorageError::Io)?; + Ok(()) + } + + async fn delete_bucket(&self, name: &str) -> StorageResult<()> { + let bucket_path = self.require_bucket(name)?; + let (has_objects, has_versions, has_multipart) = + self.check_bucket_contents_sync(&bucket_path); + if has_objects { + return Err(StorageError::BucketNotEmpty(name.to_string())); + } + if has_versions { + return Err(StorageError::BucketNotEmpty( + "Bucket contains archived object versions".to_string(), + )); + } + if has_multipart { + return Err(StorageError::BucketNotEmpty( + "Bucket has active multipart uploads".to_string(), + )); + } + + Self::remove_tree(&bucket_path); + Self::remove_tree(&self.system_bucket_root(name)); + Self::remove_tree(&self.multipart_bucket_root(name)); + + self.bucket_config_cache.remove(name); + self.stats_cache.remove(name); + + Ok(()) + } + + async fn bucket_exists(&self, name: &str) -> StorageResult { + Ok(self.bucket_path(name).exists()) + } + + async fn bucket_stats(&self, name: &str) -> StorageResult { + self.bucket_stats_sync(name) + } + + async fn put_object( + &self, + bucket: &str, + key: &str, + mut stream: AsyncReadStream, + metadata: Option>, + ) -> StorageResult { + let mut data = Vec::new(); + stream + .read_to_end(&mut data) + .await + .map_err(StorageError::Io)?; + self.put_object_sync(bucket, key, &data, metadata) + } + + async fn get_object( + &self, + bucket: &str, + key: &str, + ) -> StorageResult<(ObjectMeta, AsyncReadStream)> { + let path = self.object_path(bucket, key)?; + if !path.is_file() { + return Err(StorageError::ObjectNotFound { + bucket: bucket.to_string(), + key: key.to_string(), + }); + } + + let meta = std::fs::metadata(&path).map_err(StorageError::Io)?; + let mtime = meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0); + let lm = Utc + .timestamp_opt(mtime as i64, ((mtime % 1.0) * 1_000_000_000.0) as u32) + .single() + .unwrap_or_else(Utc::now); + + let stored_meta = self.read_metadata_sync(bucket, key); + let mut obj = ObjectMeta::new(key.to_string(), meta.len(), lm); + obj.etag = stored_meta.get("__etag__").cloned(); + obj.content_type = stored_meta.get("__content_type__").cloned(); + obj.metadata = stored_meta + .into_iter() + .filter(|(k, _)| !k.starts_with("__")) + .collect(); + + let file = tokio::fs::File::open(&path).await.map_err(StorageError::Io)?; + let stream: AsyncReadStream = Box::pin(file); + Ok((obj, stream)) + } + + async fn get_object_path(&self, bucket: &str, key: &str) -> StorageResult { + let path = self.object_path(bucket, key)?; + if !path.is_file() { + return Err(StorageError::ObjectNotFound { + bucket: bucket.to_string(), + key: key.to_string(), + }); + } + Ok(path) + } + + async fn head_object(&self, bucket: &str, key: &str) -> StorageResult { + let path = self.object_path(bucket, key)?; + if !path.is_file() { + return Err(StorageError::ObjectNotFound { + bucket: bucket.to_string(), + key: key.to_string(), + }); + } + + let meta = std::fs::metadata(&path).map_err(StorageError::Io)?; + let mtime = meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0); + let lm = Utc + .timestamp_opt(mtime as i64, ((mtime % 1.0) * 1_000_000_000.0) as u32) + .single() + .unwrap_or_else(Utc::now); + + let stored_meta = self.read_metadata_sync(bucket, key); + let mut obj = ObjectMeta::new(key.to_string(), meta.len(), lm); + obj.etag = stored_meta.get("__etag__").cloned(); + obj.content_type = stored_meta.get("__content_type__").cloned(); + obj.metadata = stored_meta + .into_iter() + .filter(|(k, _)| !k.starts_with("__")) + .collect(); + Ok(obj) + } + + async fn delete_object(&self, bucket: &str, key: &str) -> StorageResult<()> { + let bucket_path = self.require_bucket(bucket)?; + let path = self.object_path(bucket, key)?; + if !path.exists() { + return Ok(()); + } + + let versioning_enabled = self.read_bucket_config_sync(bucket).versioning_enabled; + if versioning_enabled { + self.archive_current_version_sync(bucket, key, "delete") + .map_err(StorageError::Io)?; + } + + Self::safe_unlink(&path).map_err(StorageError::Io)?; + self.delete_metadata_sync(bucket, key) + .map_err(StorageError::Io)?; + + Self::cleanup_empty_parents(&path, &bucket_path); + Ok(()) + } + + async fn copy_object( + &self, + src_bucket: &str, + src_key: &str, + dst_bucket: &str, + dst_key: &str, + ) -> StorageResult { + let src_path = self.object_path(src_bucket, src_key)?; + if !src_path.is_file() { + return Err(StorageError::ObjectNotFound { + bucket: src_bucket.to_string(), + key: src_key.to_string(), + }); + } + + let data = std::fs::read(&src_path).map_err(StorageError::Io)?; + let src_metadata = self.read_metadata_sync(src_bucket, src_key); + self.put_object_sync(dst_bucket, dst_key, &data, Some(src_metadata)) + } + + async fn get_object_metadata( + &self, + bucket: &str, + key: &str, + ) -> StorageResult> { + Ok(self.read_metadata_sync(bucket, key)) + } + + async fn put_object_metadata( + &self, + bucket: &str, + key: &str, + metadata: &HashMap, + ) -> StorageResult<()> { + let mut entry = self.read_index_entry_sync(bucket, key).unwrap_or_default(); + let meta_map: serde_json::Map = metadata + .iter() + .map(|(k, v)| (k.clone(), Value::String(v.clone()))) + .collect(); + entry.insert("metadata".to_string(), Value::Object(meta_map)); + self.write_index_entry_sync(bucket, key, &entry) + .map_err(StorageError::Io)?; + Ok(()) + } + + async fn list_objects( + &self, + bucket: &str, + params: &ListParams, + ) -> StorageResult { + self.list_objects_sync(bucket, params) + } + + async fn list_objects_shallow( + &self, + bucket: &str, + params: &ShallowListParams, + ) -> StorageResult { + self.list_objects_shallow_sync(bucket, params) + } + + async fn initiate_multipart( + &self, + bucket: &str, + key: &str, + metadata: Option>, + ) -> StorageResult { + self.require_bucket(bucket)?; + self.validate_key(key)?; + + let upload_id = Uuid::new_v4().to_string().replace('-', ""); + let upload_dir = self.multipart_bucket_root(bucket).join(&upload_id); + std::fs::create_dir_all(&upload_dir).map_err(StorageError::Io)?; + + let manifest = serde_json::json!({ + "upload_id": upload_id, + "object_key": key, + "metadata": metadata.unwrap_or_default(), + "created_at": Utc::now().to_rfc3339(), + "parts": {} + }); + + let manifest_path = upload_dir.join(MANIFEST_FILE); + Self::atomic_write_json_sync(&manifest_path, &manifest, true) + .map_err(StorageError::Io)?; + + Ok(upload_id) + } + + async fn upload_part( + &self, + bucket: &str, + upload_id: &str, + part_number: u32, + mut stream: AsyncReadStream, + ) -> StorageResult { + let upload_dir = self.multipart_bucket_root(bucket).join(upload_id); + let manifest_path = upload_dir.join(MANIFEST_FILE); + if !manifest_path.exists() { + return Err(StorageError::UploadNotFound(upload_id.to_string())); + } + + let mut data = Vec::new(); + stream + .read_to_end(&mut data) + .await + .map_err(StorageError::Io)?; + + let mut hasher = Md5::new(); + hasher.update(&data); + let etag = format!("{:x}", hasher.finalize()); + + let part_file = upload_dir.join(format!("part-{:05}.part", part_number)); + std::fs::write(&part_file, &data).map_err(StorageError::Io)?; + + let lock_path = upload_dir.join(".manifest.lock"); + let lock = self.get_meta_index_lock(&lock_path.to_string_lossy()); + let _guard = lock.lock(); + + let manifest_content = + std::fs::read_to_string(&manifest_path).map_err(StorageError::Io)?; + let mut manifest: Value = + serde_json::from_str(&manifest_content).map_err(StorageError::Json)?; + + if let Some(parts) = manifest.get_mut("parts").and_then(|p| p.as_object_mut()) { + parts.insert( + part_number.to_string(), + serde_json::json!({ + "etag": etag, + "size": data.len(), + "filename": format!("part-{:05}.part", part_number), + }), + ); + } + + Self::atomic_write_json_sync(&manifest_path, &manifest, true) + .map_err(StorageError::Io)?; + + Ok(etag) + } + + async fn complete_multipart( + &self, + bucket: &str, + upload_id: &str, + parts: &[PartInfo], + ) -> StorageResult { + let upload_dir = self.multipart_bucket_root(bucket).join(upload_id); + let manifest_path = upload_dir.join(MANIFEST_FILE); + if !manifest_path.exists() { + return Err(StorageError::UploadNotFound(upload_id.to_string())); + } + + let manifest_content = + std::fs::read_to_string(&manifest_path).map_err(StorageError::Io)?; + let manifest: Value = + serde_json::from_str(&manifest_content).map_err(StorageError::Json)?; + + let object_key = manifest + .get("object_key") + .and_then(|v| v.as_str()) + .ok_or_else(|| StorageError::Internal("Missing object_key in manifest".to_string()))? + .to_string(); + + let metadata: HashMap = manifest + .get("metadata") + .and_then(|v| serde_json::from_value(v.clone()).ok()) + .unwrap_or_default(); + + let mut combined_data = Vec::new(); + for part_info in parts { + let part_file = upload_dir.join(format!("part-{:05}.part", part_info.part_number)); + if !part_file.exists() { + return Err(StorageError::InvalidObjectKey(format!( + "Part {} not found", + part_info.part_number + ))); + } + let part_data = std::fs::read(&part_file).map_err(StorageError::Io)?; + combined_data.extend_from_slice(&part_data); + } + + let result = self.put_object_sync(bucket, &object_key, &combined_data, Some(metadata))?; + + let _ = std::fs::remove_dir_all(&upload_dir); + + Ok(result) + } + + async fn abort_multipart(&self, bucket: &str, upload_id: &str) -> StorageResult<()> { + let upload_dir = self.multipart_bucket_root(bucket).join(upload_id); + if upload_dir.exists() { + std::fs::remove_dir_all(&upload_dir).map_err(StorageError::Io)?; + } + Ok(()) + } + + async fn list_parts(&self, bucket: &str, upload_id: &str) -> StorageResult> { + let upload_dir = self.multipart_bucket_root(bucket).join(upload_id); + let manifest_path = upload_dir.join(MANIFEST_FILE); + if !manifest_path.exists() { + return Err(StorageError::UploadNotFound(upload_id.to_string())); + } + + let content = std::fs::read_to_string(&manifest_path).map_err(StorageError::Io)?; + let manifest: Value = serde_json::from_str(&content).map_err(StorageError::Json)?; + + let mut parts = Vec::new(); + if let Some(Value::Object(parts_map)) = manifest.get("parts") { + for (num_str, info) in parts_map { + let part_number: u32 = num_str.parse().unwrap_or(0); + let etag = info + .get("etag") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let size = info.get("size").and_then(|v| v.as_u64()).unwrap_or(0); + parts.push(PartMeta { + part_number, + etag, + size, + last_modified: None, + }); + } + } + + parts.sort_by_key(|p| p.part_number); + Ok(parts) + } + + async fn list_multipart_uploads( + &self, + bucket: &str, + ) -> StorageResult> { + let uploads_root = self.multipart_bucket_root(bucket); + if !uploads_root.exists() { + return Ok(Vec::new()); + } + + let mut uploads = Vec::new(); + let entries = std::fs::read_dir(&uploads_root).map_err(StorageError::Io)?; + for entry in entries.flatten() { + if !entry.file_type().map(|ft| ft.is_dir()).unwrap_or(false) { + continue; + } + let upload_id = entry.file_name().to_string_lossy().to_string(); + let manifest_path = entry.path().join(MANIFEST_FILE); + if !manifest_path.exists() { + continue; + } + if let Ok(content) = std::fs::read_to_string(&manifest_path) { + if let Ok(manifest) = serde_json::from_str::(&content) { + let key = manifest + .get("object_key") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let created = manifest + .get("created_at") + .and_then(|v| v.as_str()) + .and_then(|s| DateTime::parse_from_rfc3339(s).ok()) + .map(|d| d.with_timezone(&Utc)) + .unwrap_or_else(Utc::now); + uploads.push(MultipartUploadInfo { + upload_id, + key, + initiated: created, + }); + } + } + } + + Ok(uploads) + } + + async fn get_bucket_config(&self, bucket: &str) -> StorageResult { + self.require_bucket(bucket)?; + Ok(self.read_bucket_config_sync(bucket)) + } + + async fn set_bucket_config(&self, bucket: &str, config: &BucketConfig) -> StorageResult<()> { + self.require_bucket(bucket)?; + self.write_bucket_config_sync(bucket, config) + .map_err(StorageError::Io) + } + + async fn is_versioning_enabled(&self, bucket: &str) -> StorageResult { + Ok(self.read_bucket_config_sync(bucket).versioning_enabled) + } + + async fn set_versioning(&self, bucket: &str, enabled: bool) -> StorageResult<()> { + self.require_bucket(bucket)?; + let mut config = self.read_bucket_config_sync(bucket); + config.versioning_enabled = enabled; + self.write_bucket_config_sync(bucket, &config) + .map_err(StorageError::Io) + } + + async fn list_object_versions( + &self, + bucket: &str, + key: &str, + ) -> StorageResult> { + self.require_bucket(bucket)?; + let version_dir = self.version_dir(bucket, key); + if !version_dir.exists() { + return Ok(Vec::new()); + } + + let mut versions = Vec::new(); + let entries = std::fs::read_dir(&version_dir).map_err(StorageError::Io)?; + for entry in entries.flatten() { + let name = entry.file_name().to_string_lossy().to_string(); + if !name.ends_with(".json") { + continue; + } + if let Ok(content) = std::fs::read_to_string(entry.path()) { + if let Ok(record) = serde_json::from_str::(&content) { + let version_id = record + .get("version_id") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let size = record.get("size").and_then(|v| v.as_u64()).unwrap_or(0); + let archived_at = record + .get("archived_at") + .and_then(|v| v.as_str()) + .and_then(|s| DateTime::parse_from_rfc3339(s).ok()) + .map(|d| d.with_timezone(&Utc)) + .unwrap_or_else(Utc::now); + let etag = record + .get("etag") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + + versions.push(VersionInfo { + version_id, + key: key.to_string(), + size, + last_modified: archived_at, + etag, + is_latest: false, + }); + } + } + } + + versions.sort_by(|a, b| b.last_modified.cmp(&a.last_modified)); + if let Some(first) = versions.first_mut() { + first.is_latest = true; + } + + Ok(versions) + } + + async fn get_object_tags( + &self, + bucket: &str, + key: &str, + ) -> StorageResult> { + self.require_bucket(bucket)?; + let obj_path = self.object_path(bucket, key)?; + if !obj_path.exists() { + return Err(StorageError::ObjectNotFound { bucket: bucket.to_string(), key: key.to_string() }); + } + + let entry = self.read_index_entry_sync(bucket, key); + if let Some(entry) = entry { + if let Some(tags_val) = entry.get("tags") { + if let Ok(tags) = serde_json::from_value::>(tags_val.clone()) { + return Ok(tags); + } + } + } + Ok(Vec::new()) + } + + async fn set_object_tags( + &self, + bucket: &str, + key: &str, + tags: &[Tag], + ) -> StorageResult<()> { + self.require_bucket(bucket)?; + let obj_path = self.object_path(bucket, key)?; + if !obj_path.exists() { + return Err(StorageError::ObjectNotFound { bucket: bucket.to_string(), key: key.to_string() }); + } + + let mut entry = self.read_index_entry_sync(bucket, key).unwrap_or_default(); + if tags.is_empty() { + entry.remove("tags"); + } else { + entry.insert( + "tags".to_string(), + serde_json::to_value(tags).unwrap_or(Value::Null), + ); + } + + self.write_index_entry_sync(bucket, key, &entry) + .map_err(StorageError::Io)?; + Ok(()) + } + + async fn delete_object_tags( + &self, + bucket: &str, + key: &str, + ) -> StorageResult<()> { + self.set_object_tags(bucket, key, &[]).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::traits::StorageEngine; + + fn create_test_backend() -> (tempfile::TempDir, FsStorageBackend) { + let dir = tempfile::tempdir().unwrap(); + let backend = FsStorageBackend::new(dir.path().to_path_buf()); + (dir, backend) + } + + #[tokio::test] + async fn test_create_and_list_buckets() { + let (_dir, backend) = create_test_backend(); + backend.create_bucket("test-bucket").await.unwrap(); + let buckets = backend.list_buckets().await.unwrap(); + assert_eq!(buckets.len(), 1); + assert_eq!(buckets[0].name, "test-bucket"); + } + + #[tokio::test] + async fn test_bucket_exists() { + let (_dir, backend) = create_test_backend(); + assert!(!backend.bucket_exists("test-bucket").await.unwrap()); + backend.create_bucket("test-bucket").await.unwrap(); + assert!(backend.bucket_exists("test-bucket").await.unwrap()); + } + + #[tokio::test] + async fn test_delete_bucket() { + let (_dir, backend) = create_test_backend(); + backend.create_bucket("test-bucket").await.unwrap(); + backend.delete_bucket("test-bucket").await.unwrap(); + assert!(!backend.bucket_exists("test-bucket").await.unwrap()); + } + + #[tokio::test] + async fn test_delete_nonempty_bucket_fails() { + let (_dir, backend) = create_test_backend(); + backend.create_bucket("test-bucket").await.unwrap(); + let data: AsyncReadStream = Box::pin(std::io::Cursor::new(b"hello".to_vec())); + backend + .put_object("test-bucket", "file.txt", data, None) + .await + .unwrap(); + let result = backend.delete_bucket("test-bucket").await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_put_and_get_object() { + let (_dir, backend) = create_test_backend(); + backend.create_bucket("test-bucket").await.unwrap(); + + let data: AsyncReadStream = Box::pin(std::io::Cursor::new(b"hello world".to_vec())); + let meta = backend + .put_object("test-bucket", "greeting.txt", data, None) + .await + .unwrap(); + assert_eq!(meta.size, 11); + assert!(meta.etag.is_some()); + + let (obj, mut stream) = backend.get_object("test-bucket", "greeting.txt").await.unwrap(); + assert_eq!(obj.size, 11); + let mut buf = Vec::new(); + stream.read_to_end(&mut buf).await.unwrap(); + assert_eq!(buf, b"hello world"); + } + + #[tokio::test] + async fn test_head_object() { + let (_dir, backend) = create_test_backend(); + backend.create_bucket("test-bucket").await.unwrap(); + + let data: AsyncReadStream = Box::pin(std::io::Cursor::new(b"test data".to_vec())); + backend + .put_object("test-bucket", "file.txt", data, None) + .await + .unwrap(); + + let meta = backend.head_object("test-bucket", "file.txt").await.unwrap(); + assert_eq!(meta.size, 9); + assert!(meta.etag.is_some()); + } + + #[tokio::test] + async fn test_delete_object() { + let (_dir, backend) = create_test_backend(); + backend.create_bucket("test-bucket").await.unwrap(); + + let data: AsyncReadStream = Box::pin(std::io::Cursor::new(b"delete me".to_vec())); + backend + .put_object("test-bucket", "file.txt", data, None) + .await + .unwrap(); + + backend.delete_object("test-bucket", "file.txt").await.unwrap(); + let result = backend.head_object("test-bucket", "file.txt").await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_put_object_with_metadata() { + let (_dir, backend) = create_test_backend(); + backend.create_bucket("test-bucket").await.unwrap(); + + let mut user_meta = HashMap::new(); + user_meta.insert("x-amz-meta-custom".to_string(), "myvalue".to_string()); + user_meta.insert("__content_type__".to_string(), "text/plain".to_string()); + + let data: AsyncReadStream = Box::pin(std::io::Cursor::new(b"hello".to_vec())); + backend + .put_object("test-bucket", "file.txt", data, Some(user_meta)) + .await + .unwrap(); + + let stored = backend + .get_object_metadata("test-bucket", "file.txt") + .await + .unwrap(); + assert_eq!(stored.get("x-amz-meta-custom").unwrap(), "myvalue"); + assert_eq!(stored.get("__content_type__").unwrap(), "text/plain"); + assert!(stored.contains_key("__etag__")); + } + + #[tokio::test] + async fn test_list_objects() { + let (_dir, backend) = create_test_backend(); + backend.create_bucket("test-bucket").await.unwrap(); + + for name in &["a.txt", "b.txt", "c.txt"] { + let data: AsyncReadStream = Box::pin(std::io::Cursor::new(b"x".to_vec())); + backend + .put_object("test-bucket", name, data, None) + .await + .unwrap(); + } + + let result = backend + .list_objects("test-bucket", &ListParams::default()) + .await + .unwrap(); + assert_eq!(result.objects.len(), 3); + assert_eq!(result.objects[0].key, "a.txt"); + assert_eq!(result.objects[1].key, "b.txt"); + assert_eq!(result.objects[2].key, "c.txt"); + assert!(!result.is_truncated); + } + + #[tokio::test] + async fn test_list_objects_with_prefix() { + let (_dir, backend) = create_test_backend(); + backend.create_bucket("test-bucket").await.unwrap(); + + for name in &["docs/a.txt", "docs/b.txt", "images/c.png"] { + let data: AsyncReadStream = Box::pin(std::io::Cursor::new(b"x".to_vec())); + backend + .put_object("test-bucket", name, data, None) + .await + .unwrap(); + } + + let params = ListParams { + prefix: Some("docs/".to_string()), + ..Default::default() + }; + let result = backend.list_objects("test-bucket", ¶ms).await.unwrap(); + assert_eq!(result.objects.len(), 2); + } + + #[tokio::test] + async fn test_list_objects_pagination() { + let (_dir, backend) = create_test_backend(); + backend.create_bucket("test-bucket").await.unwrap(); + + for i in 0..5 { + let data: AsyncReadStream = Box::pin(std::io::Cursor::new(b"x".to_vec())); + backend + .put_object("test-bucket", &format!("file{}.txt", i), data, None) + .await + .unwrap(); + } + + let params = ListParams { + max_keys: 2, + ..Default::default() + }; + let result = backend.list_objects("test-bucket", ¶ms).await.unwrap(); + assert_eq!(result.objects.len(), 2); + assert!(result.is_truncated); + assert!(result.next_continuation_token.is_some()); + + let params2 = ListParams { + max_keys: 2, + continuation_token: result.next_continuation_token, + ..Default::default() + }; + let result2 = backend.list_objects("test-bucket", ¶ms2).await.unwrap(); + assert_eq!(result2.objects.len(), 2); + assert!(result2.is_truncated); + } + + #[tokio::test] + async fn test_copy_object() { + let (_dir, backend) = create_test_backend(); + backend.create_bucket("src-bucket").await.unwrap(); + backend.create_bucket("dst-bucket").await.unwrap(); + + let data: AsyncReadStream = Box::pin(std::io::Cursor::new(b"copy me".to_vec())); + backend + .put_object("src-bucket", "original.txt", data, None) + .await + .unwrap(); + + backend + .copy_object("src-bucket", "original.txt", "dst-bucket", "copied.txt") + .await + .unwrap(); + + let (_, mut stream) = backend + .get_object("dst-bucket", "copied.txt") + .await + .unwrap(); + let mut buf = Vec::new(); + stream.read_to_end(&mut buf).await.unwrap(); + assert_eq!(buf, b"copy me"); + } + + #[tokio::test] + async fn test_multipart_upload() { + let (_dir, backend) = create_test_backend(); + backend.create_bucket("test-bucket").await.unwrap(); + + let upload_id = backend + .initiate_multipart("test-bucket", "big-file.bin", None) + .await + .unwrap(); + + let part1: AsyncReadStream = Box::pin(std::io::Cursor::new(b"part1-data".to_vec())); + let etag1 = backend + .upload_part("test-bucket", &upload_id, 1, part1) + .await + .unwrap(); + + let part2: AsyncReadStream = Box::pin(std::io::Cursor::new(b"part2-data".to_vec())); + let etag2 = backend + .upload_part("test-bucket", &upload_id, 2, part2) + .await + .unwrap(); + + let parts = vec![ + PartInfo { + part_number: 1, + etag: etag1, + }, + PartInfo { + part_number: 2, + etag: etag2, + }, + ]; + + let result = backend + .complete_multipart("test-bucket", &upload_id, &parts) + .await + .unwrap(); + assert_eq!(result.size, 20); + + let (_, mut stream) = backend + .get_object("test-bucket", "big-file.bin") + .await + .unwrap(); + let mut buf = Vec::new(); + stream.read_to_end(&mut buf).await.unwrap(); + assert_eq!(buf, b"part1-datapart2-data"); + } + + #[tokio::test] + async fn test_versioning() { + let (_dir, backend) = create_test_backend(); + backend.create_bucket("test-bucket").await.unwrap(); + backend.set_versioning("test-bucket", true).await.unwrap(); + + let data1: AsyncReadStream = Box::pin(std::io::Cursor::new(b"version1".to_vec())); + backend + .put_object("test-bucket", "file.txt", data1, None) + .await + .unwrap(); + + let data2: AsyncReadStream = Box::pin(std::io::Cursor::new(b"version2".to_vec())); + backend + .put_object("test-bucket", "file.txt", data2, None) + .await + .unwrap(); + + let versions = backend + .list_object_versions("test-bucket", "file.txt") + .await + .unwrap(); + assert_eq!(versions.len(), 1); + assert_eq!(versions[0].size, 8); + } + + #[tokio::test] + async fn test_invalid_bucket_name() { + let (_dir, backend) = create_test_backend(); + let result = backend.create_bucket("AB").await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_bucket_stats() { + let (_dir, backend) = create_test_backend(); + backend.create_bucket("test-bucket").await.unwrap(); + + let data: AsyncReadStream = Box::pin(std::io::Cursor::new(b"hello".to_vec())); + backend + .put_object("test-bucket", "file.txt", data, None) + .await + .unwrap(); + + let stats = backend.bucket_stats("test-bucket").await.unwrap(); + assert_eq!(stats.objects, 1); + assert_eq!(stats.bytes, 5); + } +} diff --git a/myfsio-engine/crates/myfsio-storage/src/lib.rs b/myfsio-engine/crates/myfsio-storage/src/lib.rs new file mode 100644 index 0000000..f18411e --- /dev/null +++ b/myfsio-engine/crates/myfsio-storage/src/lib.rs @@ -0,0 +1,4 @@ +pub mod validation; +pub mod traits; +pub mod error; +pub mod fs_backend; diff --git a/myfsio-engine/crates/myfsio-storage/src/traits.rs b/myfsio-engine/crates/myfsio-storage/src/traits.rs new file mode 100644 index 0000000..1b27184 --- /dev/null +++ b/myfsio-engine/crates/myfsio-storage/src/traits.rs @@ -0,0 +1,125 @@ +use crate::error::StorageError; +use myfsio_common::types::*; +use std::collections::HashMap; +use std::path::PathBuf; +use std::pin::Pin; +use tokio::io::AsyncRead; + +pub type StorageResult = Result; +pub type AsyncReadStream = Pin>; + +#[allow(async_fn_in_trait)] +pub trait StorageEngine: Send + Sync { + async fn list_buckets(&self) -> StorageResult>; + async fn create_bucket(&self, name: &str) -> StorageResult<()>; + async fn delete_bucket(&self, name: &str) -> StorageResult<()>; + async fn bucket_exists(&self, name: &str) -> StorageResult; + async fn bucket_stats(&self, name: &str) -> StorageResult; + + async fn put_object( + &self, + bucket: &str, + key: &str, + stream: AsyncReadStream, + metadata: Option>, + ) -> StorageResult; + + async fn get_object(&self, bucket: &str, key: &str) -> StorageResult<(ObjectMeta, AsyncReadStream)>; + + async fn get_object_path(&self, bucket: &str, key: &str) -> StorageResult; + + async fn head_object(&self, bucket: &str, key: &str) -> StorageResult; + + async fn delete_object(&self, bucket: &str, key: &str) -> StorageResult<()>; + + async fn copy_object( + &self, + src_bucket: &str, + src_key: &str, + dst_bucket: &str, + dst_key: &str, + ) -> StorageResult; + + async fn get_object_metadata( + &self, + bucket: &str, + key: &str, + ) -> StorageResult>; + + async fn put_object_metadata( + &self, + bucket: &str, + key: &str, + metadata: &HashMap, + ) -> StorageResult<()>; + + async fn list_objects(&self, bucket: &str, params: &ListParams) -> StorageResult; + + async fn list_objects_shallow( + &self, + bucket: &str, + params: &ShallowListParams, + ) -> StorageResult; + + async fn initiate_multipart( + &self, + bucket: &str, + key: &str, + metadata: Option>, + ) -> StorageResult; + + async fn upload_part( + &self, + bucket: &str, + upload_id: &str, + part_number: u32, + stream: AsyncReadStream, + ) -> StorageResult; + + async fn complete_multipart( + &self, + bucket: &str, + upload_id: &str, + parts: &[PartInfo], + ) -> StorageResult; + + async fn abort_multipart(&self, bucket: &str, upload_id: &str) -> StorageResult<()>; + + async fn list_parts(&self, bucket: &str, upload_id: &str) -> StorageResult>; + + async fn list_multipart_uploads( + &self, + bucket: &str, + ) -> StorageResult>; + + async fn get_bucket_config(&self, bucket: &str) -> StorageResult; + async fn set_bucket_config(&self, bucket: &str, config: &BucketConfig) -> StorageResult<()>; + + async fn is_versioning_enabled(&self, bucket: &str) -> StorageResult; + async fn set_versioning(&self, bucket: &str, enabled: bool) -> StorageResult<()>; + + async fn list_object_versions( + &self, + bucket: &str, + key: &str, + ) -> StorageResult>; + + async fn get_object_tags( + &self, + bucket: &str, + key: &str, + ) -> StorageResult>; + + async fn set_object_tags( + &self, + bucket: &str, + key: &str, + tags: &[Tag], + ) -> StorageResult<()>; + + async fn delete_object_tags( + &self, + bucket: &str, + key: &str, + ) -> StorageResult<()>; +} diff --git a/myfsio-engine/crates/myfsio-storage/src/validation.rs b/myfsio-engine/crates/myfsio-storage/src/validation.rs new file mode 100644 index 0000000..99004d8 --- /dev/null +++ b/myfsio-engine/crates/myfsio-storage/src/validation.rs @@ -0,0 +1,194 @@ +use std::sync::LazyLock; +use unicode_normalization::UnicodeNormalization; + +const WINDOWS_RESERVED: &[&str] = &[ + "CON", "PRN", "AUX", "NUL", "COM0", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", + "COM8", "COM9", "LPT0", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", + "LPT9", +]; + +const WINDOWS_ILLEGAL_CHARS: &[char] = &['<', '>', ':', '"', '/', '\\', '|', '?', '*']; + +const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"]; +const SYSTEM_ROOT: &str = ".myfsio.sys"; + +static IP_REGEX: LazyLock = + LazyLock::new(|| regex::Regex::new(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$").unwrap()); + +pub fn validate_object_key( + object_key: &str, + max_length_bytes: usize, + is_windows: bool, + reserved_prefixes: Option<&[&str]>, +) -> Option { + if object_key.is_empty() { + return Some("Object key required".to_string()); + } + + if object_key.contains('\0') { + return Some("Object key contains null bytes".to_string()); + } + + let normalized: String = object_key.nfc().collect(); + + if normalized.len() > max_length_bytes { + return Some(format!( + "Object key exceeds maximum length of {} bytes", + max_length_bytes + )); + } + + if normalized.starts_with('/') || normalized.starts_with('\\') { + return Some("Object key cannot start with a slash".to_string()); + } + + let parts: Vec<&str> = if cfg!(windows) || is_windows { + normalized.split(['/', '\\']).collect() + } else { + normalized.split('/').collect() + }; + + for part in &parts { + if part.is_empty() { + continue; + } + + if *part == ".." { + return Some("Object key contains parent directory references".to_string()); + } + + if *part == "." { + return Some("Object key contains invalid segments".to_string()); + } + + if part.chars().any(|c| (c as u32) < 32) { + return Some("Object key contains control characters".to_string()); + } + + if is_windows { + if part.chars().any(|c| WINDOWS_ILLEGAL_CHARS.contains(&c)) { + return Some( + "Object key contains characters not supported on Windows filesystems" + .to_string(), + ); + } + if part.ends_with(' ') || part.ends_with('.') { + return Some( + "Object key segments cannot end with spaces or periods on Windows".to_string(), + ); + } + let trimmed = part.trim_end_matches(['.', ' ']).to_uppercase(); + if WINDOWS_RESERVED.contains(&trimmed.as_str()) { + return Some(format!("Invalid filename segment: {}", part)); + } + } + } + + let non_empty_parts: Vec<&str> = parts.iter().filter(|p| !p.is_empty()).copied().collect(); + if let Some(top) = non_empty_parts.first() { + if INTERNAL_FOLDERS.contains(top) || *top == SYSTEM_ROOT { + return Some("Object key uses a reserved prefix".to_string()); + } + + if let Some(prefixes) = reserved_prefixes { + for prefix in prefixes { + if *top == *prefix { + return Some("Object key uses a reserved prefix".to_string()); + } + } + } + } + + None +} + +pub fn validate_bucket_name(bucket_name: &str) -> Option { + let len = bucket_name.len(); + if len < 3 || len > 63 { + return Some("Bucket name must be between 3 and 63 characters".to_string()); + } + + let bytes = bucket_name.as_bytes(); + if !bytes[0].is_ascii_lowercase() && !bytes[0].is_ascii_digit() { + return Some( + "Bucket name must start and end with a lowercase letter or digit".to_string(), + ); + } + if !bytes[len - 1].is_ascii_lowercase() && !bytes[len - 1].is_ascii_digit() { + return Some( + "Bucket name must start and end with a lowercase letter or digit".to_string(), + ); + } + + for &b in bytes { + if !b.is_ascii_lowercase() && !b.is_ascii_digit() && b != b'.' && b != b'-' { + return Some( + "Bucket name can only contain lowercase letters, digits, dots, and hyphens" + .to_string(), + ); + } + } + + if bucket_name.contains("..") { + return Some("Bucket name must not contain consecutive periods".to_string()); + } + + if IP_REGEX.is_match(bucket_name) { + return Some("Bucket name must not be formatted as an IP address".to_string()); + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_valid_bucket_names() { + assert!(validate_bucket_name("my-bucket").is_none()); + assert!(validate_bucket_name("test123").is_none()); + assert!(validate_bucket_name("my.bucket.name").is_none()); + } + + #[test] + fn test_invalid_bucket_names() { + assert!(validate_bucket_name("ab").is_some()); + assert!(validate_bucket_name("My-Bucket").is_some()); + assert!(validate_bucket_name("-bucket").is_some()); + assert!(validate_bucket_name("bucket-").is_some()); + assert!(validate_bucket_name("my..bucket").is_some()); + assert!(validate_bucket_name("192.168.1.1").is_some()); + } + + #[test] + fn test_valid_object_keys() { + assert!(validate_object_key("file.txt", 1024, false, None).is_none()); + assert!(validate_object_key("path/to/file.txt", 1024, false, None).is_none()); + assert!(validate_object_key("a", 1024, false, None).is_none()); + } + + #[test] + fn test_invalid_object_keys() { + assert!(validate_object_key("", 1024, false, None).is_some()); + assert!(validate_object_key("/leading-slash", 1024, false, None).is_some()); + assert!(validate_object_key("path/../escape", 1024, false, None).is_some()); + assert!(validate_object_key(".myfsio.sys/secret", 1024, false, None).is_some()); + assert!(validate_object_key(".meta/data", 1024, false, None).is_some()); + } + + #[test] + fn test_object_key_max_length() { + let long_key = "a".repeat(1025); + assert!(validate_object_key(&long_key, 1024, false, None).is_some()); + let ok_key = "a".repeat(1024); + assert!(validate_object_key(&ok_key, 1024, false, None).is_none()); + } + + #[test] + fn test_windows_validation() { + assert!(validate_object_key("CON", 1024, true, None).is_some()); + assert!(validate_object_key("file String { + let mut writer = Writer::new(Cursor::new(Vec::new())); + writer + .create_element(tag) + .write_text_content(quick_xml::events::BytesText::new(text)) + .unwrap(); + String::from_utf8(writer.into_inner().into_inner()).unwrap() +} diff --git a/myfsio-engine/crates/myfsio-xml/src/request.rs b/myfsio-engine/crates/myfsio-xml/src/request.rs new file mode 100644 index 0000000..f24a5ac --- /dev/null +++ b/myfsio-engine/crates/myfsio-xml/src/request.rs @@ -0,0 +1,159 @@ +use quick_xml::events::Event; +use quick_xml::Reader; + +#[derive(Debug, Default)] +pub struct DeleteObjectsRequest { + pub objects: Vec, + pub quiet: bool, +} + +#[derive(Debug)] +pub struct ObjectIdentifier { + pub key: String, + pub version_id: Option, +} + +#[derive(Debug, Default)] +pub struct CompleteMultipartUpload { + pub parts: Vec, +} + +#[derive(Debug)] +pub struct CompletedPart { + pub part_number: u32, + pub etag: String, +} + +pub fn parse_complete_multipart_upload(xml: &str) -> Result { + let mut reader = Reader::from_str(xml); + let mut result = CompleteMultipartUpload::default(); + let mut buf = Vec::new(); + let mut current_tag = String::new(); + let mut part_number: Option = None; + let mut etag: Option = None; + let mut in_part = false; + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(ref e)) => { + let name = String::from_utf8_lossy(e.name().as_ref()).to_string(); + current_tag = name.clone(); + if name == "Part" { + in_part = true; + part_number = None; + etag = None; + } + } + Ok(Event::Text(ref e)) => { + if in_part { + let text = e.unescape().map_err(|e| e.to_string())?.to_string(); + match current_tag.as_str() { + "PartNumber" => { + part_number = Some(text.trim().parse().map_err(|e: std::num::ParseIntError| e.to_string())?); + } + "ETag" => { + etag = Some(text.trim().trim_matches('"').to_string()); + } + _ => {} + } + } + } + Ok(Event::End(ref e)) => { + let name = String::from_utf8_lossy(e.name().as_ref()).to_string(); + if name == "Part" && in_part { + if let (Some(pn), Some(et)) = (part_number.take(), etag.take()) { + result.parts.push(CompletedPart { + part_number: pn, + etag: et, + }); + } + in_part = false; + } + } + Ok(Event::Eof) => break, + Err(e) => return Err(format!("XML parse error: {}", e)), + _ => {} + } + buf.clear(); + } + + result.parts.sort_by_key(|p| p.part_number); + Ok(result) +} + +pub fn parse_delete_objects(xml: &str) -> Result { + let mut reader = Reader::from_str(xml); + let mut result = DeleteObjectsRequest::default(); + let mut buf = Vec::new(); + let mut current_tag = String::new(); + let mut current_key: Option = None; + let mut current_version_id: Option = None; + let mut in_object = false; + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(ref e)) => { + let name = String::from_utf8_lossy(e.name().as_ref()).to_string(); + current_tag = name.clone(); + if name == "Object" { + in_object = true; + current_key = None; + current_version_id = None; + } + } + Ok(Event::Text(ref e)) => { + let text = e.unescape().map_err(|e| e.to_string())?.to_string(); + match current_tag.as_str() { + "Key" if in_object => { + current_key = Some(text.trim().to_string()); + } + "VersionId" if in_object => { + current_version_id = Some(text.trim().to_string()); + } + "Quiet" => { + result.quiet = text.trim() == "true"; + } + _ => {} + } + } + Ok(Event::End(ref e)) => { + let name = String::from_utf8_lossy(e.name().as_ref()).to_string(); + if name == "Object" && in_object { + if let Some(key) = current_key.take() { + result.objects.push(ObjectIdentifier { + key, + version_id: current_version_id.take(), + }); + } + in_object = false; + } + } + Ok(Event::Eof) => break, + Err(e) => return Err(format!("XML parse error: {}", e)), + _ => {} + } + buf.clear(); + } + + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_complete_multipart() { + let xml = r#" + 2"etag2" + 1"etag1" + "#; + + let result = parse_complete_multipart_upload(xml).unwrap(); + assert_eq!(result.parts.len(), 2); + assert_eq!(result.parts[0].part_number, 1); + assert_eq!(result.parts[0].etag, "etag1"); + assert_eq!(result.parts[1].part_number, 2); + assert_eq!(result.parts[1].etag, "etag2"); + } +} diff --git a/myfsio-engine/crates/myfsio-xml/src/response.rs b/myfsio-engine/crates/myfsio-xml/src/response.rs new file mode 100644 index 0000000..18b086d --- /dev/null +++ b/myfsio-engine/crates/myfsio-xml/src/response.rs @@ -0,0 +1,269 @@ +use myfsio_common::types::{BucketMeta, ObjectMeta}; +use quick_xml::events::{BytesDecl, BytesEnd, BytesStart, BytesText, Event}; +use quick_xml::Writer; +use std::io::Cursor; + +pub fn list_buckets_xml(owner_id: &str, owner_name: &str, buckets: &[BucketMeta]) -> String { + let mut writer = Writer::new(Cursor::new(Vec::new())); + + writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap(); + + let start = BytesStart::new("ListAllMyBucketsResult") + .with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]); + writer.write_event(Event::Start(start)).unwrap(); + + writer.write_event(Event::Start(BytesStart::new("Owner"))).unwrap(); + write_text_element(&mut writer, "ID", owner_id); + write_text_element(&mut writer, "DisplayName", owner_name); + writer.write_event(Event::End(BytesEnd::new("Owner"))).unwrap(); + + writer.write_event(Event::Start(BytesStart::new("Buckets"))).unwrap(); + for bucket in buckets { + writer.write_event(Event::Start(BytesStart::new("Bucket"))).unwrap(); + write_text_element(&mut writer, "Name", &bucket.name); + write_text_element(&mut writer, "CreationDate", &bucket.creation_date.to_rfc3339()); + writer.write_event(Event::End(BytesEnd::new("Bucket"))).unwrap(); + } + writer.write_event(Event::End(BytesEnd::new("Buckets"))).unwrap(); + + writer.write_event(Event::End(BytesEnd::new("ListAllMyBucketsResult"))).unwrap(); + + String::from_utf8(writer.into_inner().into_inner()).unwrap() +} + +pub fn list_objects_v2_xml( + bucket_name: &str, + prefix: &str, + delimiter: &str, + max_keys: usize, + objects: &[ObjectMeta], + common_prefixes: &[String], + is_truncated: bool, + continuation_token: Option<&str>, + next_continuation_token: Option<&str>, + key_count: usize, +) -> String { + let mut writer = Writer::new(Cursor::new(Vec::new())); + + writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap(); + + let start = BytesStart::new("ListBucketResult") + .with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]); + writer.write_event(Event::Start(start)).unwrap(); + + write_text_element(&mut writer, "Name", bucket_name); + write_text_element(&mut writer, "Prefix", prefix); + if !delimiter.is_empty() { + write_text_element(&mut writer, "Delimiter", delimiter); + } + write_text_element(&mut writer, "MaxKeys", &max_keys.to_string()); + write_text_element(&mut writer, "KeyCount", &key_count.to_string()); + write_text_element(&mut writer, "IsTruncated", &is_truncated.to_string()); + + if let Some(token) = continuation_token { + write_text_element(&mut writer, "ContinuationToken", token); + } + if let Some(token) = next_continuation_token { + write_text_element(&mut writer, "NextContinuationToken", token); + } + + for obj in objects { + writer.write_event(Event::Start(BytesStart::new("Contents"))).unwrap(); + write_text_element(&mut writer, "Key", &obj.key); + write_text_element(&mut writer, "LastModified", &obj.last_modified.to_rfc3339()); + if let Some(ref etag) = obj.etag { + write_text_element(&mut writer, "ETag", &format!("\"{}\"", etag)); + } + write_text_element(&mut writer, "Size", &obj.size.to_string()); + write_text_element(&mut writer, "StorageClass", obj.storage_class.as_deref().unwrap_or("STANDARD")); + writer.write_event(Event::End(BytesEnd::new("Contents"))).unwrap(); + } + + for prefix in common_prefixes { + writer.write_event(Event::Start(BytesStart::new("CommonPrefixes"))).unwrap(); + write_text_element(&mut writer, "Prefix", prefix); + writer.write_event(Event::End(BytesEnd::new("CommonPrefixes"))).unwrap(); + } + + writer.write_event(Event::End(BytesEnd::new("ListBucketResult"))).unwrap(); + + String::from_utf8(writer.into_inner().into_inner()).unwrap() +} + +fn write_text_element(writer: &mut Writer>>, tag: &str, text: &str) { + writer.write_event(Event::Start(BytesStart::new(tag))).unwrap(); + writer.write_event(Event::Text(BytesText::new(text))).unwrap(); + writer.write_event(Event::End(BytesEnd::new(tag))).unwrap(); +} + +pub fn initiate_multipart_upload_xml(bucket: &str, key: &str, upload_id: &str) -> String { + let mut writer = Writer::new(Cursor::new(Vec::new())); + writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap(); + + let start = BytesStart::new("InitiateMultipartUploadResult") + .with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]); + writer.write_event(Event::Start(start)).unwrap(); + write_text_element(&mut writer, "Bucket", bucket); + write_text_element(&mut writer, "Key", key); + write_text_element(&mut writer, "UploadId", upload_id); + writer.write_event(Event::End(BytesEnd::new("InitiateMultipartUploadResult"))).unwrap(); + + String::from_utf8(writer.into_inner().into_inner()).unwrap() +} + +pub fn complete_multipart_upload_xml( + bucket: &str, + key: &str, + etag: &str, + location: &str, +) -> String { + let mut writer = Writer::new(Cursor::new(Vec::new())); + writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap(); + + let start = BytesStart::new("CompleteMultipartUploadResult") + .with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]); + writer.write_event(Event::Start(start)).unwrap(); + write_text_element(&mut writer, "Location", location); + write_text_element(&mut writer, "Bucket", bucket); + write_text_element(&mut writer, "Key", key); + write_text_element(&mut writer, "ETag", &format!("\"{}\"", etag)); + writer.write_event(Event::End(BytesEnd::new("CompleteMultipartUploadResult"))).unwrap(); + + String::from_utf8(writer.into_inner().into_inner()).unwrap() +} + +pub fn copy_object_result_xml(etag: &str, last_modified: &str) -> String { + let mut writer = Writer::new(Cursor::new(Vec::new())); + writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap(); + + let start = BytesStart::new("CopyObjectResult") + .with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]); + writer.write_event(Event::Start(start)).unwrap(); + write_text_element(&mut writer, "ETag", &format!("\"{}\"", etag)); + write_text_element(&mut writer, "LastModified", last_modified); + writer.write_event(Event::End(BytesEnd::new("CopyObjectResult"))).unwrap(); + + String::from_utf8(writer.into_inner().into_inner()).unwrap() +} + +pub fn delete_result_xml( + deleted: &[(String, Option)], + errors: &[(String, String, String)], + quiet: bool, +) -> String { + let mut writer = Writer::new(Cursor::new(Vec::new())); + writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap(); + + let start = BytesStart::new("DeleteResult") + .with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]); + writer.write_event(Event::Start(start)).unwrap(); + + if !quiet { + for (key, version_id) in deleted { + writer.write_event(Event::Start(BytesStart::new("Deleted"))).unwrap(); + write_text_element(&mut writer, "Key", key); + if let Some(vid) = version_id { + write_text_element(&mut writer, "VersionId", vid); + } + writer.write_event(Event::End(BytesEnd::new("Deleted"))).unwrap(); + } + } + + for (key, code, message) in errors { + writer.write_event(Event::Start(BytesStart::new("Error"))).unwrap(); + write_text_element(&mut writer, "Key", key); + write_text_element(&mut writer, "Code", code); + write_text_element(&mut writer, "Message", message); + writer.write_event(Event::End(BytesEnd::new("Error"))).unwrap(); + } + + writer.write_event(Event::End(BytesEnd::new("DeleteResult"))).unwrap(); + + String::from_utf8(writer.into_inner().into_inner()).unwrap() +} + +pub fn list_multipart_uploads_xml( + bucket: &str, + uploads: &[myfsio_common::types::MultipartUploadInfo], +) -> String { + let mut writer = Writer::new(Cursor::new(Vec::new())); + writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap(); + + let start = BytesStart::new("ListMultipartUploadsResult") + .with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]); + writer.write_event(Event::Start(start)).unwrap(); + write_text_element(&mut writer, "Bucket", bucket); + + for upload in uploads { + writer.write_event(Event::Start(BytesStart::new("Upload"))).unwrap(); + write_text_element(&mut writer, "Key", &upload.key); + write_text_element(&mut writer, "UploadId", &upload.upload_id); + write_text_element(&mut writer, "Initiated", &upload.initiated.to_rfc3339()); + writer.write_event(Event::End(BytesEnd::new("Upload"))).unwrap(); + } + + writer.write_event(Event::End(BytesEnd::new("ListMultipartUploadsResult"))).unwrap(); + + String::from_utf8(writer.into_inner().into_inner()).unwrap() +} + +pub fn list_parts_xml( + bucket: &str, + key: &str, + upload_id: &str, + parts: &[myfsio_common::types::PartMeta], +) -> String { + let mut writer = Writer::new(Cursor::new(Vec::new())); + writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))).unwrap(); + + let start = BytesStart::new("ListPartsResult") + .with_attributes([("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/")]); + writer.write_event(Event::Start(start)).unwrap(); + write_text_element(&mut writer, "Bucket", bucket); + write_text_element(&mut writer, "Key", key); + write_text_element(&mut writer, "UploadId", upload_id); + + for part in parts { + writer.write_event(Event::Start(BytesStart::new("Part"))).unwrap(); + write_text_element(&mut writer, "PartNumber", &part.part_number.to_string()); + write_text_element(&mut writer, "ETag", &format!("\"{}\"", part.etag)); + write_text_element(&mut writer, "Size", &part.size.to_string()); + if let Some(ref lm) = part.last_modified { + write_text_element(&mut writer, "LastModified", &lm.to_rfc3339()); + } + writer.write_event(Event::End(BytesEnd::new("Part"))).unwrap(); + } + + writer.write_event(Event::End(BytesEnd::new("ListPartsResult"))).unwrap(); + + String::from_utf8(writer.into_inner().into_inner()).unwrap() +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + + #[test] + fn test_list_buckets_xml() { + let buckets = vec![BucketMeta { + name: "test-bucket".to_string(), + creation_date: Utc::now(), + }]; + let xml = list_buckets_xml("owner-id", "owner-name", &buckets); + assert!(xml.contains("test-bucket")); + assert!(xml.contains("owner-id")); + assert!(xml.contains("ListAllMyBucketsResult")); + } + + #[test] + fn test_list_objects_v2_xml() { + let objects = vec![ObjectMeta::new("file.txt".to_string(), 1024, Utc::now())]; + let xml = list_objects_v2_xml( + "my-bucket", "", "/", 1000, &objects, &[], false, None, None, 1, + ); + assert!(xml.contains("file.txt")); + assert!(xml.contains("1024")); + assert!(xml.contains("false")); + } +} diff --git a/run.py b/run.py index 40ab540..51a7006 100644 --- a/run.py +++ b/run.py @@ -5,6 +5,7 @@ import argparse import atexit import os import signal +import subprocess import sys import warnings import multiprocessing @@ -74,6 +75,48 @@ def _serve_granian(target: str, port: int, config: Optional[AppConfig] = None) - server.serve() +def _find_rust_binary() -> Optional[Path]: + candidates = [ + Path(__file__).parent / "myfsio-engine" / "target" / "release" / "myfsio-server.exe", + Path(__file__).parent / "myfsio-engine" / "target" / "release" / "myfsio-server", + Path(__file__).parent / "myfsio-engine" / "target" / "debug" / "myfsio-server.exe", + Path(__file__).parent / "myfsio-engine" / "target" / "debug" / "myfsio-server", + ] + for p in candidates: + if p.exists(): + return p + return None + + +def serve_rust_api(port: int, config: Optional[AppConfig] = None) -> None: + binary = _find_rust_binary() + if binary is None: + print("ERROR: Rust engine binary not found. Build it first:") + print(" cd myfsio-engine && cargo build --release") + sys.exit(1) + + env = os.environ.copy() + env["PORT"] = str(port) + env["HOST"] = _server_host() + if config: + env["STORAGE_ROOT"] = str(config.storage_root) + env["AWS_REGION"] = config.aws_region + if config.secret_key: + env["SECRET_KEY"] = config.secret_key + env.setdefault("ENCRYPTION_ENABLED", str(config.encryption_enabled).lower()) + env.setdefault("KMS_ENABLED", str(config.kms_enabled).lower()) + env.setdefault("LIFECYCLE_ENABLED", str(config.lifecycle_enabled).lower()) + env.setdefault("RUST_LOG", "info") + + print(f"Starting Rust S3 engine: {binary}") + proc = subprocess.Popen([str(binary)], env=env) + try: + proc.wait() + except KeyboardInterrupt: + proc.terminate() + proc.wait(timeout=5) + + def serve_api(port: int, prod: bool = False, config: Optional[AppConfig] = None) -> None: if prod: _serve_granian("app:create_api_app", port, config) @@ -227,6 +270,7 @@ if __name__ == "__main__": parser.add_argument("--ui-port", type=int, default=5100) parser.add_argument("--prod", action="store_true", help="Run in production mode using Granian") parser.add_argument("--dev", action="store_true", help="Force development mode (Flask dev server)") + parser.add_argument("--engine", choices=["python", "rust"], default="python", help="API engine: python (Flask) or rust (myfsio-engine)") parser.add_argument("--check-config", action="store_true", help="Validate configuration and exit") parser.add_argument("--show-config", action="store_true", help="Show configuration summary and exit") parser.add_argument("--reset-cred", action="store_true", help="Reset admin credentials and exit") @@ -280,9 +324,17 @@ if __name__ == "__main__": else: print("Running in development mode (Flask dev server)") + use_rust = args.engine == "rust" + if args.mode in {"api", "both"}: - print(f"Starting API server on port {args.api_port}...") - api_proc = Process(target=serve_api, args=(args.api_port, prod_mode, config)) + if use_rust: + print(f"Starting Rust API engine on port {args.api_port}...") + else: + print(f"Starting API server on port {args.api_port}...") + if use_rust: + api_proc = Process(target=serve_rust_api, args=(args.api_port, config)) + else: + api_proc = Process(target=serve_api, args=(args.api_port, prod_mode, config)) api_proc.start() else: api_proc = None