From ebc315c1cc85b4c8c5729653129771ee4d27e7a0 Mon Sep 17 00:00:00 2001 From: kqjy Date: Sun, 18 Jan 2026 21:35:39 +0800 Subject: [PATCH] Fix routing conflicts: move admin endpoints to reserved paths --- Dockerfile | 2 +- README.md | 16 ++---- app/__init__.py | 2 +- app/s3_api.py | 55 +++------------------ app/ui.py | 62 ++++++++++++------------ docs.md | 28 +++++------ templates/docs.html | 35 +++++--------- tests/test_api.py | 115 ++++++++++++++++++-------------------------- 8 files changed, 119 insertions(+), 196 deletions(-) diff --git a/Dockerfile b/Dockerfile index b74c3ff..74c6efb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,6 +32,6 @@ ENV APP_HOST=0.0.0.0 \ FLASK_DEBUG=0 HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ - CMD python -c "import requests; requests.get('http://localhost:5000/healthz', timeout=2)" + CMD python -c "import requests; requests.get('http://localhost:5000/myfsio/health', timeout=2)" CMD ["./docker-entrypoint.sh"] diff --git a/README.md b/README.md index 05b2671..f1e7951 100644 --- a/README.md +++ b/README.md @@ -149,19 +149,13 @@ All endpoints require AWS Signature Version 4 authentication unless using presig | `POST` | `//?uploadId=X` | Complete multipart upload | | `DELETE` | `//?uploadId=X` | Abort multipart upload | -### Presigned URLs +### Bucket Policies (S3-compatible) | Method | Endpoint | Description | |--------|----------|-------------| -| `POST` | `/presign//` | Generate presigned URL | - -### Bucket Policies - -| Method | Endpoint | Description | -|--------|----------|-------------| -| `GET` | `/bucket-policy/` | Get bucket policy | -| `PUT` | `/bucket-policy/` | Set bucket policy | -| `DELETE` | `/bucket-policy/` | Delete bucket policy | +| `GET` | `/?policy` | Get bucket policy | +| `PUT` | `/?policy` | Set bucket policy | +| `DELETE` | `/?policy` | Delete bucket policy | ### Versioning @@ -175,7 +169,7 @@ All endpoints require AWS Signature Version 4 authentication unless using presig | Method | Endpoint | Description | |--------|----------|-------------| -| `GET` | `/healthz` | Health check endpoint | +| `GET` | `/myfsio/health` | Health check endpoint | ## IAM & Access Control diff --git a/app/__init__.py b/app/__init__.py index eac24c7..ac059c7 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -278,7 +278,7 @@ def create_app( return render_template("404.html"), 404 return error - @app.get("/healthz") + @app.get("/myfsio/health") def healthcheck() -> Dict[str, str]: return {"status": "ok"} diff --git a/app/s3_api.py b/app/s3_api.py index b184b35..77fa81e 100644 --- a/app/s3_api.py +++ b/app/s3_api.py @@ -590,6 +590,7 @@ def _generate_presigned_url( bucket_name: str, object_key: str, expires_in: int, + api_base_url: str | None = None, ) -> str: region = current_app.config["AWS_REGION"] service = current_app.config["AWS_SERVICE"] @@ -610,7 +611,7 @@ def _generate_presigned_url( } canonical_query = _encode_query_params(query_params) - api_base = current_app.config.get("API_BASE_URL") + api_base = api_base_url or current_app.config.get("API_BASE_URL") if api_base: parsed = urlparse(api_base) host = parsed.netloc @@ -940,6 +941,7 @@ def _maybe_handle_bucket_subresource(bucket_name: str) -> Response | None: "notification": _bucket_notification_handler, "logging": _bucket_logging_handler, "uploads": _bucket_uploads_handler, + "policy": _bucket_policy_handler, } requested = [key for key in handlers if key in request.args] if not requested: @@ -2642,9 +2644,9 @@ def _list_parts(bucket_name: str, object_key: str) -> Response: return _xml_response(root) -@s3_api_bp.route("/bucket-policy/", methods=["GET", "PUT", "DELETE"]) -@limiter.limit("30 per minute") -def bucket_policy_handler(bucket_name: str) -> Response: +def _bucket_policy_handler(bucket_name: str) -> Response: + if request.method not in {"GET", "PUT", "DELETE"}: + return _method_not_allowed(["GET", "PUT", "DELETE"]) principal, error = _require_principal() if error: return error @@ -2676,51 +2678,6 @@ def bucket_policy_handler(bucket_name: str) -> Response: return Response(status=204) -@s3_api_bp.post("/presign//") -@limiter.limit("45 per minute") -def presign_object(bucket_name: str, object_key: str): - payload = request.get_json(silent=True) or {} - method = str(payload.get("method", "GET")).upper() - allowed_methods = {"GET", "PUT", "DELETE"} - if method not in allowed_methods: - return _error_response("InvalidRequest", "Method must be GET, PUT, or DELETE", 400) - try: - expires = int(payload.get("expires_in", 900)) - except (TypeError, ValueError): - return _error_response("InvalidRequest", "expires_in must be an integer", 400) - expires = max(1, min(expires, 7 * 24 * 3600)) - action = "read" if method == "GET" else ("delete" if method == "DELETE" else "write") - principal, error = _require_principal() - if error: - return error - try: - _authorize_action(principal, bucket_name, action, object_key=object_key) - except IamError as exc: - return _error_response("AccessDenied", str(exc), 403) - storage = _storage() - if not storage.bucket_exists(bucket_name): - return _error_response("NoSuchBucket", "Bucket does not exist", 404) - if action != "write": - try: - storage.get_object_path(bucket_name, object_key) - except StorageError: - return _error_response("NoSuchKey", "Object not found", 404) - secret = _iam().secret_for_key(principal.access_key) - url = _generate_presigned_url( - principal=principal, - secret_key=secret, - method=method, - bucket_name=bucket_name, - object_key=object_key, - expires_in=expires, - ) - current_app.logger.info( - "Presigned URL generated", - extra={"bucket": bucket_name, "key": object_key, "method": method}, - ) - return jsonify({"url": url, "method": method, "expires_in": expires}) - - @s3_api_bp.route("/", methods=["HEAD"]) @limiter.limit("100 per minute") def head_bucket(bucket_name: str) -> Response: diff --git a/app/ui.py b/app/ui.py index e3fa6fd..7e86c61 100644 --- a/app/ui.py +++ b/app/ui.py @@ -36,6 +36,7 @@ from .extensions import limiter, csrf from .iam import IamError from .kms import KMSManager from .replication import ReplicationManager, ReplicationRule +from .s3_api import _generate_presigned_url from .secret_store import EphemeralSecretStore from .storage import ObjectStorage, StorageError @@ -1135,42 +1136,43 @@ def object_presign(bucket_name: str, object_key: str): principal = _current_principal() payload = request.get_json(silent=True) or {} method = str(payload.get("method", "GET")).upper() + allowed_methods = {"GET", "PUT", "DELETE"} + if method not in allowed_methods: + return jsonify({"error": "Method must be GET, PUT, or DELETE"}), 400 action = "read" if method == "GET" else ("delete" if method == "DELETE" else "write") try: _authorize_ui(principal, bucket_name, action, object_key=object_key) except IamError as exc: return jsonify({"error": str(exc)}), 403 - + try: + expires = int(payload.get("expires_in", 900)) + except (TypeError, ValueError): + return jsonify({"error": "expires_in must be an integer"}), 400 + expires = max(1, min(expires, 7 * 24 * 3600)) + storage = _storage() + if not storage.bucket_exists(bucket_name): + return jsonify({"error": "Bucket does not exist"}), 404 + if action != "write": + try: + storage.get_object_path(bucket_name, object_key) + except StorageError: + return jsonify({"error": "Object not found"}), 404 + secret = _iam().secret_for_key(principal.access_key) api_base = current_app.config.get("API_BASE_URL") or "http://127.0.0.1:5000" - api_base = api_base.rstrip("/") - encoded_key = quote(object_key, safe="/") - url = f"{api_base}/presign/{bucket_name}/{encoded_key}" - - parsed_api = urlparse(api_base) - headers = _api_headers() - headers["X-Forwarded-Host"] = parsed_api.netloc or "127.0.0.1:5000" - headers["X-Forwarded-Proto"] = parsed_api.scheme or "http" - headers["X-Forwarded-For"] = request.remote_addr or "127.0.0.1" - - try: - response = requests.post(url, headers=headers, json=payload, timeout=5) - except requests.RequestException as exc: - return jsonify({"error": f"API unavailable: {exc}"}), 502 - try: - body = response.json() - except ValueError: - text = response.text or "" - if text.strip().startswith("<"): - import xml.etree.ElementTree as ET - try: - root = ET.fromstring(text) - message = root.findtext(".//Message") or root.findtext(".//Code") or "Unknown S3 error" - body = {"error": message} - except ET.ParseError: - body = {"error": text or "API returned an empty response"} - else: - body = {"error": text or "API returned an empty response"} - return jsonify(body), response.status_code + url = _generate_presigned_url( + principal=principal, + secret_key=secret, + method=method, + bucket_name=bucket_name, + object_key=object_key, + expires_in=expires, + api_base_url=api_base, + ) + current_app.logger.info( + "Presigned URL generated", + extra={"bucket": bucket_name, "key": object_key, "method": method}, + ) + return jsonify({"url": url, "method": method, "expires_in": expires}) @ui_bp.get("/buckets//objects//metadata") diff --git a/docs.md b/docs.md index 01fc802..a5c131d 100644 --- a/docs.md +++ b/docs.md @@ -122,7 +122,7 @@ With these volumes attached you can rebuild/restart the container without losing ### Versioning -The repo now tracks a human-friendly release string inside `app/version.py` (see the `APP_VERSION` constant). Edit that value whenever you cut a release. The constant flows into Flask as `APP_VERSION` and is exposed via `GET /healthz`, so you can monitor deployments or surface it in UIs. +The repo now tracks a human-friendly release string inside `app/version.py` (see the `APP_VERSION` constant). Edit that value whenever you cut a release. The constant flows into Flask as `APP_VERSION` and is exposed via `GET /myfsio/health`, so you can monitor deployments or surface it in UIs. ## 3. Configuration Reference @@ -277,14 +277,14 @@ The application automatically trusts these headers to generate correct presigned ### Version Checking The application version is tracked in `app/version.py` and exposed via: -- **Health endpoint:** `GET /healthz` returns JSON with `version` field +- **Health endpoint:** `GET /myfsio/health` returns JSON with `version` field - **Metrics dashboard:** Navigate to `/ui/metrics` to see the running version in the System Status card To check your current version: ```bash # API health endpoint -curl http://localhost:5000/healthz +curl http://localhost:5000/myfsio/health # Or inspect version.py directly cat app/version.py | grep APP_VERSION @@ -377,7 +377,7 @@ docker run -d \ myfsio:latest # 5. Verify health -curl http://localhost:5000/healthz +curl http://localhost:5000/myfsio/health ``` ### Version Compatibility Checks @@ -502,7 +502,7 @@ docker run -d \ myfsio:0.1.3 # specify previous version tag # 3. Verify -curl http://localhost:5000/healthz +curl http://localhost:5000/myfsio/health ``` #### Emergency Config Restore @@ -528,7 +528,7 @@ For production environments requiring zero downtime: APP_PORT=5001 UI_PORT=5101 python run.py & # 2. Health check new instance -curl http://localhost:5001/healthz +curl http://localhost:5001/myfsio/health # 3. Update load balancer to route to new ports @@ -544,7 +544,7 @@ After any update, verify functionality: ```bash # 1. Health check -curl http://localhost:5000/healthz +curl http://localhost:5000/myfsio/health # 2. Login to UI open http://localhost:5100/ui @@ -588,7 +588,7 @@ APP_PID=$! # Wait and health check sleep 5 -if curl -f http://localhost:5000/healthz; then +if curl -f http://localhost:5000/myfsio/health; then echo "Update successful!" else echo "Health check failed, rolling back..." @@ -860,7 +860,7 @@ A request is allowed only if: ### Editing via CLI ```bash -curl -X PUT http://127.0.0.1:5000/bucket-policy/test \ +curl -X PUT "http://127.0.0.1:5000/test?policy" \ -H "Content-Type: application/json" \ -H "X-Access-Key: ..." -H "X-Secret-Key: ..." \ -d '{ @@ -923,9 +923,8 @@ Drag files directly onto the objects table to upload them to the current bucket ## 6. Presigned URLs - Trigger from the UI using the **Presign** button after selecting an object. -- Or call `POST /presign//` with JSON `{ "method": "GET", "expires_in": 900 }`. - Supported methods: `GET`, `PUT`, `DELETE`; expiration must be `1..604800` seconds. -- The service signs requests using the caller’s IAM credentials and enforces bucket policies both when issuing and when the presigned URL is used. +- The service signs requests using the caller's IAM credentials and enforces bucket policies both when issuing and when the presigned URL is used. - Legacy share links have been removed; presigned URLs now handle both private and public workflows. ### Multipart Upload Example @@ -1314,10 +1313,9 @@ GET / # List objects PUT // # Upload object GET // # Download object DELETE // # Delete object -POST /presign// # Generate SigV4 URL -GET /bucket-policy/ # Fetch policy -PUT /bucket-policy/ # Upsert policy -DELETE /bucket-policy/ # Delete policy +GET /?policy # Fetch policy +PUT /?policy # Upsert policy +DELETE /?policy # Delete policy GET /?quota # Get bucket quota PUT /?quota # Set bucket quota (admin only) ``` diff --git a/templates/docs.html b/templates/docs.html index 56e5553..26b43fc 100644 --- a/templates/docs.html +++ b/templates/docs.html @@ -375,11 +375,8 @@ curl -X PUT {{ api_base }}/demo/notes.txt \ -H "X-Secret-Key: <secret_key>" \ --data-binary @notes.txt -curl -X POST {{ api_base }}/presign/demo/notes.txt \ - -H "Content-Type: application/json" \ - -H "X-Access-Key: <access_key>" \ - -H "X-Secret-Key: <secret_key>" \ - -d '{"method":"GET", "expires_in": 900}' +# Presigned URLs are generated via the UI +# Use the "Presign" button in the object browser @@ -437,13 +434,8 @@ curl -X POST {{ api_base }}/presign/demo/notes.txt \ GET/PUT/DELETE - /bucket-policy/<bucket> - Fetch, upsert, or remove a bucket policy. - - - POST - /presign/<bucket>/<key> - Generate SigV4 URLs for GET/PUT/DELETE with custom expiry. + /<bucket>?policy + Fetch, upsert, or remove a bucket policy (S3-compatible). @@ -542,17 +534,16 @@ s3.complete_multipart_upload( )

Presigned URLs for Sharing

-
# Generate a download link valid for 15 minutes
-curl -X POST "{{ api_base }}/presign/mybucket/photo.jpg" \
-  -H "Content-Type: application/json" \
-  -H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
-  -d '{"method": "GET", "expires_in": 900}'
+
# Generate presigned URLs via the UI:
+# 1. Navigate to your bucket in the object browser
+# 2. Select the object you want to share
+# 3. Click the "Presign" button
+# 4. Choose method (GET/PUT/DELETE) and expiration time
+# 5. Copy the generated URL
 
-# Generate an upload link (PUT) valid for 1 hour
-curl -X POST "{{ api_base }}/presign/mybucket/upload.bin" \
-  -H "Content-Type: application/json" \
-  -H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
-  -d '{"method": "PUT", "expires_in": 3600}'

+# Supported options:
+# - Method: GET (download), PUT (upload), DELETE (remove)
+# - Expiration: 1 second to 7 days (604800 seconds)
diff --git a/tests/test_api.py b/tests/test_api.py index b2859cb..3e95b48 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,3 @@ -from urllib.parse import urlsplit - - def test_bucket_and_object_lifecycle(client, signer): headers = signer("PUT", "/photos") response = client.put("/photos", headers=headers) @@ -105,7 +102,7 @@ def test_request_id_header_present(client, signer): def test_healthcheck_returns_status(client): - response = client.get("/healthz") + response = client.get("/myfsio/health") data = response.get_json() assert response.status_code == 200 assert data["status"] == "ok" @@ -117,36 +114,20 @@ def test_missing_credentials_denied(client): assert response.status_code == 403 -def test_presign_and_bucket_policies(client, signer): - # Create bucket and object +def test_bucket_policies_deny_reads(client, signer): + import json + headers = signer("PUT", "/docs") assert client.put("/docs", headers=headers).status_code == 200 - + headers = signer("PUT", "/docs/readme.txt", body=b"content") assert client.put("/docs/readme.txt", headers=headers, data=b"content").status_code == 200 - # Generate presigned GET URL and follow it - json_body = {"method": "GET", "expires_in": 120} - # Flask test client json parameter automatically sets Content-Type and serializes body - # But for signing we need the body bytes. - import json - body_bytes = json.dumps(json_body).encode("utf-8") - headers = signer("POST", "/presign/docs/readme.txt", headers={"Content-Type": "application/json"}, body=body_bytes) - - response = client.post( - "/presign/docs/readme.txt", - headers=headers, - json=json_body, - ) + headers = signer("GET", "/docs/readme.txt") + response = client.get("/docs/readme.txt", headers=headers) assert response.status_code == 200 - presigned_url = response.get_json()["url"] - parts = urlsplit(presigned_url) - presigned_path = f"{parts.path}?{parts.query}" - download = client.get(presigned_path) - assert download.status_code == 200 - assert download.data == b"content" + assert response.data == b"content" - # Attach a deny policy for GETs policy = { "Version": "2012-10-17", "Statement": [ @@ -160,29 +141,26 @@ def test_presign_and_bucket_policies(client, signer): ], } policy_bytes = json.dumps(policy).encode("utf-8") - headers = signer("PUT", "/bucket-policy/docs", headers={"Content-Type": "application/json"}, body=policy_bytes) - assert client.put("/bucket-policy/docs", headers=headers, json=policy).status_code == 204 - - headers = signer("GET", "/bucket-policy/docs") - fetched = client.get("/bucket-policy/docs", headers=headers) + headers = signer("PUT", "/docs?policy", headers={"Content-Type": "application/json"}, body=policy_bytes) + assert client.put("/docs?policy", headers=headers, json=policy).status_code == 204 + + headers = signer("GET", "/docs?policy") + fetched = client.get("/docs?policy", headers=headers) assert fetched.status_code == 200 assert fetched.get_json()["Version"] == "2012-10-17" - # Reads are now denied by bucket policy headers = signer("GET", "/docs/readme.txt") denied = client.get("/docs/readme.txt", headers=headers) assert denied.status_code == 403 - # Presign attempts are also denied - json_body = {"method": "GET", "expires_in": 60} - body_bytes = json.dumps(json_body).encode("utf-8") - headers = signer("POST", "/presign/docs/readme.txt", headers={"Content-Type": "application/json"}, body=body_bytes) - response = client.post( - "/presign/docs/readme.txt", - headers=headers, - json=json_body, - ) - assert response.status_code == 403 + headers = signer("DELETE", "/docs?policy") + assert client.delete("/docs?policy", headers=headers).status_code == 204 + + headers = signer("DELETE", "/docs/readme.txt") + assert client.delete("/docs/readme.txt", headers=headers).status_code == 204 + + headers = signer("DELETE", "/docs") + assert client.delete("/docs", headers=headers).status_code == 204 def test_trailing_slash_returns_xml(client): @@ -193,9 +171,11 @@ def test_trailing_slash_returns_xml(client): def test_public_policy_allows_anonymous_list_and_read(client, signer): + import json + headers = signer("PUT", "/public") assert client.put("/public", headers=headers).status_code == 200 - + headers = signer("PUT", "/public/hello.txt", body=b"hi") assert client.put("/public/hello.txt", headers=headers, data=b"hi").status_code == 200 @@ -221,10 +201,9 @@ def test_public_policy_allows_anonymous_list_and_read(client, signer): }, ], } - import json policy_bytes = json.dumps(policy).encode("utf-8") - headers = signer("PUT", "/bucket-policy/public", headers={"Content-Type": "application/json"}, body=policy_bytes) - assert client.put("/bucket-policy/public", headers=headers, json=policy).status_code == 204 + headers = signer("PUT", "/public?policy", headers={"Content-Type": "application/json"}, body=policy_bytes) + assert client.put("/public?policy", headers=headers, json=policy).status_code == 204 list_response = client.get("/public") assert list_response.status_code == 200 @@ -236,18 +215,20 @@ def test_public_policy_allows_anonymous_list_and_read(client, signer): headers = signer("DELETE", "/public/hello.txt") assert client.delete("/public/hello.txt", headers=headers).status_code == 204 - - headers = signer("DELETE", "/bucket-policy/public") - assert client.delete("/bucket-policy/public", headers=headers).status_code == 204 - + + headers = signer("DELETE", "/public?policy") + assert client.delete("/public?policy", headers=headers).status_code == 204 + headers = signer("DELETE", "/public") assert client.delete("/public", headers=headers).status_code == 204 def test_principal_dict_with_object_get_only(client, signer): + import json + headers = signer("PUT", "/mixed") assert client.put("/mixed", headers=headers).status_code == 200 - + headers = signer("PUT", "/mixed/only.txt", body=b"ok") assert client.put("/mixed/only.txt", headers=headers, data=b"ok").status_code == 200 @@ -270,10 +251,9 @@ def test_principal_dict_with_object_get_only(client, signer): }, ], } - import json policy_bytes = json.dumps(policy).encode("utf-8") - headers = signer("PUT", "/bucket-policy/mixed", headers={"Content-Type": "application/json"}, body=policy_bytes) - assert client.put("/bucket-policy/mixed", headers=headers, json=policy).status_code == 204 + headers = signer("PUT", "/mixed?policy", headers={"Content-Type": "application/json"}, body=policy_bytes) + assert client.put("/mixed?policy", headers=headers, json=policy).status_code == 204 assert client.get("/mixed").status_code == 403 allowed = client.get("/mixed/only.txt") @@ -282,18 +262,20 @@ def test_principal_dict_with_object_get_only(client, signer): headers = signer("DELETE", "/mixed/only.txt") assert client.delete("/mixed/only.txt", headers=headers).status_code == 204 - - headers = signer("DELETE", "/bucket-policy/mixed") - assert client.delete("/bucket-policy/mixed", headers=headers).status_code == 204 - + + headers = signer("DELETE", "/mixed?policy") + assert client.delete("/mixed?policy", headers=headers).status_code == 204 + headers = signer("DELETE", "/mixed") assert client.delete("/mixed", headers=headers).status_code == 204 def test_bucket_policy_wildcard_resource_allows_object_get(client, signer): + import json + headers = signer("PUT", "/test") assert client.put("/test", headers=headers).status_code == 200 - + headers = signer("PUT", "/test/vid.mp4", body=b"video") assert client.put("/test/vid.mp4", headers=headers, data=b"video").status_code == 200 @@ -314,10 +296,9 @@ def test_bucket_policy_wildcard_resource_allows_object_get(client, signer): }, ], } - import json policy_bytes = json.dumps(policy).encode("utf-8") - headers = signer("PUT", "/bucket-policy/test", headers={"Content-Type": "application/json"}, body=policy_bytes) - assert client.put("/bucket-policy/test", headers=headers, json=policy).status_code == 204 + headers = signer("PUT", "/test?policy", headers={"Content-Type": "application/json"}, body=policy_bytes) + assert client.put("/test?policy", headers=headers, json=policy).status_code == 204 listing = client.get("/test") assert listing.status_code == 403 @@ -327,10 +308,10 @@ def test_bucket_policy_wildcard_resource_allows_object_get(client, signer): headers = signer("DELETE", "/test/vid.mp4") assert client.delete("/test/vid.mp4", headers=headers).status_code == 204 - - headers = signer("DELETE", "/bucket-policy/test") - assert client.delete("/bucket-policy/test", headers=headers).status_code == 204 - + + headers = signer("DELETE", "/test?policy") + assert client.delete("/test?policy", headers=headers).status_code == 204 + headers = signer("DELETE", "/test") assert client.delete("/test", headers=headers).status_code == 204