From ec5d52f20824596b4dbac0214f0c55d51f419405 Mon Sep 17 00:00:00 2001 From: kqjy Date: Sat, 22 Nov 2025 15:02:29 +0800 Subject: [PATCH] Improve and add two-way replication functionality; Update docs --- app/s3_api.py | 1 + app/ui.py | 1 + docs.md | 25 ++++++++++++++++++++++++- templates/docs.html | 16 ++++++++++++++-- 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/app/s3_api.py b/app/s3_api.py index 950990a..b080346 100644 --- a/app/s3_api.py +++ b/app/s3_api.py @@ -1043,6 +1043,7 @@ def bucket_handler(bucket_name: str) -> Response: try: storage.delete_bucket(bucket_name) _bucket_policies().delete_policy(bucket_name) + _replication_manager().delete_rule(bucket_name) except StorageError as exc: code = "BucketNotEmpty" if "not empty" in str(exc) else "NoSuchBucket" status = 409 if code == "BucketNotEmpty" else 404 diff --git a/app/ui.py b/app/ui.py index d603259..adf1f6f 100644 --- a/app/ui.py +++ b/app/ui.py @@ -500,6 +500,7 @@ def delete_bucket(bucket_name: str): _authorize_ui(principal, bucket_name, "delete") _storage().delete_bucket(bucket_name) _bucket_policies().delete_policy(bucket_name) + _replication_manager().delete_rule(bucket_name) flash(f"Bucket '{bucket_name}' removed", "success") except (StorageError, IamError) as exc: flash(_friendly_error_message(exc), "danger") diff --git a/docs.md b/docs.md index e55a737..016ffbb 100644 --- a/docs.md +++ b/docs.md @@ -77,12 +77,20 @@ The repo now tracks a human-friendly release string inside `app/version.py` (see | `SECRET_KEY` | `dev-secret-key` | Flask session key for UI auth. | | `IAM_CONFIG` | `/data/.myfsio.sys/config/iam.json` | Stores users, secrets, and inline policies. | | `BUCKET_POLICY_PATH` | `/data/.myfsio.sys/config/bucket_policies.json` | Bucket policy store (auto hot-reload). | -| `API_BASE_URL` | `http://127.0.0.1:5000` | Used by the UI to hit API endpoints (presign/policy). | +| `API_BASE_URL` | `None` | Used by the UI to hit API endpoints (presign/policy). If unset, the UI will auto-detect the host or use `X-Forwarded-*` headers. | | `AWS_REGION` | `us-east-1` | Region embedded in SigV4 credential scope. | | `AWS_SERVICE` | `s3` | Service string for SigV4. | Set env vars (or pass overrides to `create_app`) to point the servers at custom paths. +### Proxy Configuration + +If running behind a reverse proxy (e.g., Nginx, Cloudflare, or a tunnel), ensure the proxy sets the standard forwarding headers: +- `X-Forwarded-Host` +- `X-Forwarded-Proto` + +The application automatically trusts these headers to generate correct presigned URLs (e.g., `https://s3.example.com/...` instead of `http://127.0.0.1:5000/...`). Alternatively, you can explicitly set `API_BASE_URL` to your public endpoint. + ## 4. Authentication & IAM 1. On first boot, `data/.myfsio.sys/config/iam.json` is seeded with `localadmin / localadmin` that has wildcard access. @@ -262,6 +270,21 @@ Now, configure the primary instance to replicate to the target. aws --endpoint-url http://target-server:5002 s3 ls s3://backup-bucket ``` +### Bidirectional Replication (Active-Active) + +To set up two-way replication (Server A ↔ Server B): + +1. Follow the steps above to replicate **A → B**. +2. Repeat the process on Server B to replicate **B → A**: + - Create a connection on Server B pointing to Server A. + - Enable replication on the target bucket on Server B. + +**Loop Prevention**: The system automatically detects replication traffic using a custom User-Agent (`S3ReplicationAgent`). This prevents infinite loops where an object replicated from A to B is immediately replicated back to A. + +**Deletes**: Deleting an object on one server will propagate the deletion to the other server. + +**Note**: Deleting a bucket will automatically remove its associated replication configuration. + ## 7. Running Tests ```bash diff --git a/templates/docs.html b/templates/docs.html index e25c124..6f0dd40 100644 --- a/templates/docs.html +++ b/templates/docs.html @@ -290,6 +290,18 @@ s3.complete_multipart_upload( + +

Bidirectional Replication (Active-Active)

+

To set up two-way replication (Server A ↔ Server B):

+
    +
  1. Follow the steps above to replicate A → B.
    2. +
  2. Repeat the process on Server B to replicate B → A (create a connection to A, enable rule).
    3. +
+

+ Loop Prevention: The system automatically detects replication traffic using a custom User-Agent (S3ReplicationAgent). This prevents infinite loops where an object replicated from A to B is immediately replicated back to A. +
+ Deletes: Deleting an object on one server will propagate the deletion to the other server. +

@@ -330,8 +342,8 @@ s3.complete_multipart_upload( Requests hit the wrong host - API_BASE_URL not updated after tunneling/forwarding - Set API_BASE_URL in your shell or .env to match the published host. + Proxy headers missing or API_BASE_URL incorrect + Ensure your proxy sends X-Forwarded-Host/Proto headers, or explicitly set API_BASE_URL to your public domain.