23 Commits

Author SHA1 Message Date
fd8fb21517 Merge pull request 'Prepare for binary release' (#2) from next into main
Reviewed-on: #2
2025-11-22 12:33:38 +00:00
a095616569 Prepare for binary release 2025-11-22 20:32:57 +08:00
c6cbe822e1 Merge pull request 'Release v0.1.1' (#1) from next into main
Reviewed-on: #1
2025-11-22 12:31:27 +00:00
dddab6dbbc Change logging method 2025-11-22 17:47:01 +08:00
015c9cb52d Cleanup setup 2025-11-22 17:41:23 +08:00
c8b1c33118 Switch gunicorn to waitress 2025-11-22 17:20:52 +08:00
ebef3dfa57 Second test for Server Header change 2025-11-22 16:16:10 +08:00
1116353d0f Update docker-entrypoint.sh 2025-11-22 16:10:15 +08:00
e4b92a32a1 Fix and test custom server header 2025-11-22 16:09:24 +08:00
57c40dcdcc Test server header 2025-11-22 15:51:43 +08:00
7d1735a59f Fix server headers 2025-11-22 15:20:42 +08:00
9064f9d60e Fix CSRF token issue on login 2025-11-22 15:13:33 +08:00
36c08b0ac1 Update dockerfile with gunicorn for prod 2025-11-22 15:06:17 +08:00
ec5d52f208 Improve and add two-way replication functionality; Update docs 2025-11-22 15:02:29 +08:00
96de6164d1 Replication fixes 2025-11-22 14:45:21 +08:00
8c00d7bd4b Enhance replication functionalilty 2025-11-22 14:32:28 +08:00
a32d9dbd77 Fix replication corruption issue 2025-11-22 14:13:41 +08:00
fe3eacd2be Debug replication corruption issue 2025-11-22 12:56:33 +08:00
471cf5a305 Debug replication corruption issue 2025-11-22 12:11:41 +08:00
840fd176d3 Add missing CSRF tokens 2025-11-21 23:16:45 +08:00
5350d04ba5 Add missing CSRF token in connections.html 2025-11-21 23:04:56 +08:00
f2daa8a8a3 Fix IAM credentials reset causing presigned URL to fail 2025-11-21 22:32:42 +08:00
e287b59645 Fix Dockerfile permission issues 2025-11-21 22:11:38 +08:00
17 changed files with 696 additions and 100 deletions

13
.dockerignore Normal file
View File

@@ -0,0 +1,13 @@
.git
.gitignore
.venv
__pycache__
*.pyc
*.pyo
*.pyd
.pytest_cache
.coverage
htmlcov
logs
data
tmp

View File

@@ -16,9 +16,14 @@ RUN pip install --no-cache-dir -r requirements.txt
COPY . . COPY . .
# Drop privileges # Make entrypoint executable
RUN useradd -m -u 1000 myfsio \ RUN chmod +x docker-entrypoint.sh
# Create data directory and set permissions
RUN mkdir -p /app/data \
&& useradd -m -u 1000 myfsio \
&& chown -R myfsio:myfsio /app && chown -R myfsio:myfsio /app
USER myfsio USER myfsio
EXPOSE 5000 5100 EXPOSE 5000 5100
@@ -29,4 +34,4 @@ ENV APP_HOST=0.0.0.0 \
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
CMD python -c "import requests; requests.get('http://localhost:5000/healthz', timeout=2)" CMD python -c "import requests; requests.get('http://localhost:5000/healthz', timeout=2)"
CMD ["python", "run.py", "--mode", "both"] CMD ["./docker-entrypoint.sh"]

View File

@@ -2,6 +2,7 @@
from __future__ import annotations from __future__ import annotations
import logging import logging
import sys
import time import time
import uuid import uuid
from logging.handlers import RotatingFileHandler from logging.handlers import RotatingFileHandler
@@ -12,6 +13,7 @@ from typing import Any, Dict, Optional
from flask import Flask, g, has_request_context, redirect, render_template, request, url_for from flask import Flask, g, has_request_context, redirect, render_template, request, url_for
from flask_cors import CORS from flask_cors import CORS
from flask_wtf.csrf import CSRFError from flask_wtf.csrf import CSRFError
from werkzeug.middleware.proxy_fix import ProxyFix
from .bucket_policies import BucketPolicyStore from .bucket_policies import BucketPolicyStore
from .config import AppConfig from .config import AppConfig
@@ -33,7 +35,11 @@ def create_app(
"""Create and configure the Flask application.""" """Create and configure the Flask application."""
config = AppConfig.from_env(test_config) config = AppConfig.from_env(test_config)
project_root = Path(__file__).resolve().parent.parent if getattr(sys, "frozen", False):
project_root = Path(sys._MEIPASS)
else:
project_root = Path(__file__).resolve().parent.parent
app = Flask( app = Flask(
__name__, __name__,
static_folder=str(project_root / "static"), static_folder=str(project_root / "static"),
@@ -47,6 +53,9 @@ def create_app(
if app.config.get("TESTING"): if app.config.get("TESTING"):
app.config.setdefault("WTF_CSRF_ENABLED", False) app.config.setdefault("WTF_CSRF_ENABLED", False)
# Trust X-Forwarded-* headers from proxies
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1)
_configure_cors(app) _configure_cors(app)
_configure_logging(app) _configure_logging(app)
@@ -167,23 +176,33 @@ class _RequestContextFilter(logging.Filter):
def _configure_logging(app: Flask) -> None: def _configure_logging(app: Flask) -> None:
log_file = Path(app.config["LOG_FILE"])
log_file.parent.mkdir(parents=True, exist_ok=True)
handler = RotatingFileHandler(
log_file,
maxBytes=int(app.config.get("LOG_MAX_BYTES", 5 * 1024 * 1024)),
backupCount=int(app.config.get("LOG_BACKUP_COUNT", 3)),
encoding="utf-8",
)
formatter = logging.Formatter( formatter = logging.Formatter(
"%(asctime)s | %(levelname)s | %(request_id)s | %(method)s %(path)s | %(message)s" "%(asctime)s | %(levelname)s | %(request_id)s | %(method)s %(path)s | %(message)s"
) )
handler.setFormatter(formatter)
handler.addFilter(_RequestContextFilter()) # Stream Handler (stdout) - Primary for Docker
stream_handler = logging.StreamHandler(sys.stdout)
stream_handler.setFormatter(formatter)
stream_handler.addFilter(_RequestContextFilter())
logger = app.logger logger = app.logger
logger.handlers.clear() logger.handlers.clear()
logger.addHandler(handler) logger.addHandler(stream_handler)
# File Handler (optional, if configured)
if app.config.get("LOG_TO_FILE"):
log_file = Path(app.config["LOG_FILE"])
log_file.parent.mkdir(parents=True, exist_ok=True)
file_handler = RotatingFileHandler(
log_file,
maxBytes=int(app.config.get("LOG_MAX_BYTES", 5 * 1024 * 1024)),
backupCount=int(app.config.get("LOG_BACKUP_COUNT", 3)),
encoding="utf-8",
)
file_handler.setFormatter(formatter)
file_handler.addFilter(_RequestContextFilter())
logger.addHandler(file_handler)
logger.setLevel(getattr(logging, app.config.get("LOG_LEVEL", "INFO"), logging.INFO)) logger.setLevel(getattr(logging, app.config.get("LOG_LEVEL", "INFO"), logging.INFO))
@app.before_request @app.before_request
@@ -211,5 +230,4 @@ def _configure_logging(app: Flask) -> None:
}, },
) )
response.headers["X-Request-Duration-ms"] = f"{duration_ms:.2f}" response.headers["X-Request-Duration-ms"] = f"{duration_ms:.2f}"
response.headers["Server"] = "MyFISO"
return response return response

View File

@@ -4,12 +4,18 @@ from __future__ import annotations
import os import os
import secrets import secrets
import shutil import shutil
import sys
import warnings import warnings
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import Any, Dict, Optional from typing import Any, Dict, Optional
PROJECT_ROOT = Path(__file__).resolve().parent.parent if getattr(sys, "frozen", False):
# Running in a PyInstaller bundle
PROJECT_ROOT = Path(sys._MEIPASS)
else:
# Running in a normal Python environment
PROJECT_ROOT = Path(__file__).resolve().parent.parent
def _prepare_config_file(active_path: Path, legacy_path: Optional[Path] = None) -> Path: def _prepare_config_file(active_path: Path, legacy_path: Optional[Path] = None) -> Path:
@@ -39,7 +45,7 @@ class AppConfig:
secret_key: str secret_key: str
iam_config_path: Path iam_config_path: Path
bucket_policy_path: Path bucket_policy_path: Path
api_base_url: str api_base_url: Optional[str]
aws_region: str aws_region: str
aws_service: str aws_service: str
ui_enforce_bucket_policies: bool ui_enforce_bucket_policies: bool
@@ -78,11 +84,25 @@ class AppConfig:
multipart_min_part_size = int(_get("MULTIPART_MIN_PART_SIZE", 5 * 1024 * 1024)) multipart_min_part_size = int(_get("MULTIPART_MIN_PART_SIZE", 5 * 1024 * 1024))
default_secret = "dev-secret-key" default_secret = "dev-secret-key"
secret_key = str(_get("SECRET_KEY", default_secret)) secret_key = str(_get("SECRET_KEY", default_secret))
# If using default/missing secret, try to load/persist a generated one from disk
# This ensures consistency across Gunicorn workers
if not secret_key or secret_key == default_secret: if not secret_key or secret_key == default_secret:
generated = secrets.token_urlsafe(32) secret_file = storage_root / ".myfsio.sys" / "config" / ".secret"
if secret_key == default_secret: if secret_file.exists():
warnings.warn("Using insecure default SECRET_KEY. A random value has been generated; set SECRET_KEY for production", RuntimeWarning) secret_key = secret_file.read_text().strip()
secret_key = generated else:
generated = secrets.token_urlsafe(32)
if secret_key == default_secret:
warnings.warn("Using insecure default SECRET_KEY. A random value has been generated and persisted; set SECRET_KEY for production", RuntimeWarning)
try:
secret_file.parent.mkdir(parents=True, exist_ok=True)
secret_file.write_text(generated)
secret_key = generated
except OSError:
# Fallback if we can't write to disk (e.g. read-only fs)
secret_key = generated
iam_env_override = "IAM_CONFIG" in overrides or "IAM_CONFIG" in os.environ iam_env_override = "IAM_CONFIG" in overrides or "IAM_CONFIG" in os.environ
bucket_policy_override = "BUCKET_POLICY_PATH" in overrides or "BUCKET_POLICY_PATH" in os.environ bucket_policy_override = "BUCKET_POLICY_PATH" in overrides or "BUCKET_POLICY_PATH" in os.environ
@@ -100,7 +120,10 @@ class AppConfig:
bucket_policy_path, bucket_policy_path,
legacy_path=None if bucket_policy_override else PROJECT_ROOT / "data" / "bucket_policies.json", legacy_path=None if bucket_policy_override else PROJECT_ROOT / "data" / "bucket_policies.json",
) )
api_base_url = str(_get("API_BASE_URL", "http://127.0.0.1:5000")) api_base_url = _get("API_BASE_URL", None)
if api_base_url:
api_base_url = str(api_base_url)
aws_region = str(_get("AWS_REGION", "us-east-1")) aws_region = str(_get("AWS_REGION", "us-east-1"))
aws_service = str(_get("AWS_SERVICE", "s3")) aws_service = str(_get("AWS_SERVICE", "s3"))
enforce_ui_policies = str(_get("UI_ENFORCE_BUCKET_POLICIES", "0")).lower() in {"1", "true", "yes", "on"} enforce_ui_policies = str(_get("UI_ENFORCE_BUCKET_POLICIES", "0")).lower() in {"1", "true", "yes", "on"}

View File

@@ -77,10 +77,20 @@ class IamService:
self._users: Dict[str, Dict[str, Any]] = {} self._users: Dict[str, Dict[str, Any]] = {}
self._raw_config: Dict[str, Any] = {} self._raw_config: Dict[str, Any] = {}
self._failed_attempts: Dict[str, Deque[datetime]] = {} self._failed_attempts: Dict[str, Deque[datetime]] = {}
self._last_load_time = 0.0
self._load() self._load()
def _maybe_reload(self) -> None:
"""Reload configuration if the file has changed on disk."""
try:
if self.config_path.stat().st_mtime > self._last_load_time:
self._load()
except OSError:
pass
# ---------------------- authz helpers ---------------------- # ---------------------- authz helpers ----------------------
def authenticate(self, access_key: str, secret_key: str) -> Principal: def authenticate(self, access_key: str, secret_key: str) -> Principal:
self._maybe_reload()
access_key = (access_key or "").strip() access_key = (access_key or "").strip()
secret_key = (secret_key or "").strip() secret_key = (secret_key or "").strip()
if not access_key or not secret_key: if not access_key or not secret_key:
@@ -135,12 +145,14 @@ class IamService:
return int(max(0, self.auth_lockout_window.total_seconds() - elapsed)) return int(max(0, self.auth_lockout_window.total_seconds() - elapsed))
def principal_for_key(self, access_key: str) -> Principal: def principal_for_key(self, access_key: str) -> Principal:
self._maybe_reload()
record = self._users.get(access_key) record = self._users.get(access_key)
if not record: if not record:
raise IamError("Unknown access key") raise IamError("Unknown access key")
return self._build_principal(access_key, record) return self._build_principal(access_key, record)
def secret_for_key(self, access_key: str) -> str: def secret_for_key(self, access_key: str) -> str:
self._maybe_reload()
record = self._users.get(access_key) record = self._users.get(access_key)
if not record: if not record:
raise IamError("Unknown access key") raise IamError("Unknown access key")
@@ -245,6 +257,7 @@ class IamService:
# ---------------------- config helpers ---------------------- # ---------------------- config helpers ----------------------
def _load(self) -> None: def _load(self) -> None:
try: try:
self._last_load_time = self.config_path.stat().st_mtime
content = self.config_path.read_text(encoding='utf-8') content = self.config_path.read_text(encoding='utf-8')
raw = json.loads(content) raw = json.loads(content)
except FileNotFoundError: except FileNotFoundError:

View File

@@ -2,6 +2,7 @@
from __future__ import annotations from __future__ import annotations
import logging import logging
import mimetypes
import threading import threading
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass from dataclasses import dataclass
@@ -9,13 +10,17 @@ from pathlib import Path
from typing import Dict, Optional from typing import Dict, Optional
import boto3 import boto3
from botocore.config import Config
from botocore.exceptions import ClientError from botocore.exceptions import ClientError
from boto3.exceptions import S3UploadFailedError
from .connections import ConnectionStore, RemoteConnection from .connections import ConnectionStore, RemoteConnection
from .storage import ObjectStorage from .storage import ObjectStorage, StorageError
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
REPLICATION_USER_AGENT = "S3ReplicationAgent/1.0"
@dataclass @dataclass
class ReplicationRule: class ReplicationRule:
@@ -66,7 +71,26 @@ class ReplicationManager:
del self._rules[bucket_name] del self._rules[bucket_name]
self.save_rules() self.save_rules()
def trigger_replication(self, bucket_name: str, object_key: str) -> None: def create_remote_bucket(self, connection_id: str, bucket_name: str) -> None:
"""Create a bucket on the remote connection."""
connection = self.connections.get(connection_id)
if not connection:
raise ValueError(f"Connection {connection_id} not found")
try:
s3 = boto3.client(
"s3",
endpoint_url=connection.endpoint_url,
aws_access_key_id=connection.access_key,
aws_secret_access_key=connection.secret_key,
region_name=connection.region,
)
s3.create_bucket(Bucket=bucket_name)
except ClientError as e:
logger.error(f"Failed to create remote bucket {bucket_name}: {e}")
raise
def trigger_replication(self, bucket_name: str, object_key: str, action: str = "write") -> None:
rule = self.get_rule(bucket_name) rule = self.get_rule(bucket_name)
if not rule or not rule.enabled: if not rule or not rule.enabled:
return return
@@ -76,42 +100,93 @@ class ReplicationManager:
logger.warning(f"Replication skipped for {bucket_name}/{object_key}: Connection {rule.target_connection_id} not found") logger.warning(f"Replication skipped for {bucket_name}/{object_key}: Connection {rule.target_connection_id} not found")
return return
self._executor.submit(self._replicate_task, bucket_name, object_key, rule, connection) self._executor.submit(self._replicate_task, bucket_name, object_key, rule, connection, action)
def _replicate_task(self, bucket_name: str, object_key: str, rule: ReplicationRule, conn: RemoteConnection) -> None: def _replicate_task(self, bucket_name: str, object_key: str, rule: ReplicationRule, conn: RemoteConnection, action: str) -> None:
try: try:
# 1. Get local file path
# Note: We are accessing internal storage structure here.
# Ideally storage.py should expose a 'get_file_path' or we read the stream.
# For efficiency, we'll try to read the file directly if we can, or use storage.get_object
# Using boto3 to upload # Using boto3 to upload
config = Config(user_agent_extra=REPLICATION_USER_AGENT)
s3 = boto3.client( s3 = boto3.client(
"s3", "s3",
endpoint_url=conn.endpoint_url, endpoint_url=conn.endpoint_url,
aws_access_key_id=conn.access_key, aws_access_key_id=conn.access_key,
aws_secret_access_key=conn.secret_key, aws_secret_access_key=conn.secret_key,
region_name=conn.region, region_name=conn.region,
config=config,
) )
if action == "delete":
try:
s3.delete_object(Bucket=rule.target_bucket, Key=object_key)
logger.info(f"Replicated DELETE {bucket_name}/{object_key} to {conn.name} ({rule.target_bucket})")
except ClientError as e:
logger.error(f"Replication DELETE failed for {bucket_name}/{object_key}: {e}")
return
# 1. Get local file path
# Note: We are accessing internal storage structure here.
# Ideally storage.py should expose a 'get_file_path' or we read the stream.
# For efficiency, we'll try to read the file directly if we can, or use storage.get_object
# We need the file content. # We need the file content.
# Since ObjectStorage is filesystem based, let's get the stream. # Since ObjectStorage is filesystem based, let's get the stream.
# We need to be careful about closing it. # We need to be careful about closing it.
meta = self.storage.get_object_meta(bucket_name, object_key) try:
if not meta: path = self.storage.get_object_path(bucket_name, object_key)
except StorageError:
return return
with self.storage.open_object(bucket_name, object_key) as f: metadata = self.storage.get_object_metadata(bucket_name, object_key)
extra_args = {}
if meta.metadata: extra_args = {}
extra_args["Metadata"] = meta.metadata if metadata:
extra_args["Metadata"] = metadata
s3.upload_fileobj(
f, # Guess content type to prevent corruption/wrong handling
rule.target_bucket, content_type, _ = mimetypes.guess_type(path)
object_key, file_size = path.stat().st_size
ExtraArgs=extra_args
) logger.info(f"Replicating {bucket_name}/{object_key}: Size={file_size}, ContentType={content_type}")
try:
with path.open("rb") as f:
s3.put_object(
Bucket=rule.target_bucket,
Key=object_key,
Body=f,
ContentLength=file_size,
ContentType=content_type or "application/octet-stream",
Metadata=metadata or {}
)
except (ClientError, S3UploadFailedError) as e:
# Check if it's a NoSuchBucket error (either direct or wrapped)
is_no_bucket = False
if isinstance(e, ClientError):
if e.response['Error']['Code'] == 'NoSuchBucket':
is_no_bucket = True
elif isinstance(e, S3UploadFailedError):
if "NoSuchBucket" in str(e):
is_no_bucket = True
if is_no_bucket:
logger.info(f"Target bucket {rule.target_bucket} not found. Attempting to create it.")
try:
s3.create_bucket(Bucket=rule.target_bucket)
# Retry upload
with path.open("rb") as f:
s3.put_object(
Bucket=rule.target_bucket,
Key=object_key,
Body=f,
ContentLength=file_size,
ContentType=content_type or "application/octet-stream",
Metadata=metadata or {}
)
except Exception as create_err:
logger.error(f"Failed to create target bucket {rule.target_bucket}: {create_err}")
raise e # Raise original error
else:
raise e
logger.info(f"Replicated {bucket_name}/{object_key} to {conn.name} ({rule.target_bucket})") logger.info(f"Replicated {bucket_name}/{object_key} to {conn.name} ({rule.target_bucket})")

View File

@@ -8,7 +8,7 @@ import re
import uuid import uuid
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import Any, Dict from typing import Any, Dict
from urllib.parse import quote, urlencode from urllib.parse import quote, urlencode, urlparse
from xml.etree.ElementTree import Element, SubElement, tostring, fromstring, ParseError from xml.etree.ElementTree import Element, SubElement, tostring, fromstring, ParseError
from flask import Blueprint, Response, current_app, jsonify, request from flask import Blueprint, Response, current_app, jsonify, request
@@ -17,6 +17,7 @@ from werkzeug.http import http_date
from .bucket_policies import BucketPolicyStore from .bucket_policies import BucketPolicyStore
from .extensions import limiter from .extensions import limiter
from .iam import IamError, Principal from .iam import IamError, Principal
from .replication import ReplicationManager
from .storage import ObjectStorage, StorageError from .storage import ObjectStorage, StorageError
s3_api_bp = Blueprint("s3_api", __name__) s3_api_bp = Blueprint("s3_api", __name__)
@@ -31,6 +32,9 @@ def _iam():
return current_app.extensions["iam"] return current_app.extensions["iam"]
def _replication_manager() -> ReplicationManager:
return current_app.extensions["replication"]
def _bucket_policies() -> BucketPolicyStore: def _bucket_policies() -> BucketPolicyStore:
store: BucketPolicyStore = current_app.extensions["bucket_policies"] store: BucketPolicyStore = current_app.extensions["bucket_policies"]
@@ -405,7 +409,11 @@ def _canonical_headers_from_request(headers: list[str]) -> str:
lines = [] lines = []
for header in headers: for header in headers:
if header == "host": if header == "host":
value = request.host api_base = current_app.config.get("API_BASE_URL")
if api_base:
value = urlparse(api_base).netloc
else:
value = request.host
else: else:
value = request.headers.get(header, "") value = request.headers.get(header, "")
canonical_value = " ".join(value.strip().split()) if value else "" canonical_value = " ".join(value.strip().split()) if value else ""
@@ -468,7 +476,17 @@ def _generate_presigned_url(
"X-Amz-Content-Sha256": "UNSIGNED-PAYLOAD", "X-Amz-Content-Sha256": "UNSIGNED-PAYLOAD",
} }
canonical_query = _encode_query_params(query_params) canonical_query = _encode_query_params(query_params)
host = request.host
# Determine host and scheme from config or request
api_base = current_app.config.get("API_BASE_URL")
if api_base:
parsed = urlparse(api_base)
host = parsed.netloc
scheme = parsed.scheme
else:
host = request.headers.get("X-Forwarded-Host", request.host)
scheme = request.headers.get("X-Forwarded-Proto", request.scheme or "http")
canonical_headers = f"host:{host}\n" canonical_headers = f"host:{host}\n"
canonical_request = "\n".join( canonical_request = "\n".join(
[ [
@@ -492,7 +510,6 @@ def _generate_presigned_url(
signing_key = _derive_signing_key(secret_key, date_stamp, region, service) signing_key = _derive_signing_key(secret_key, date_stamp, region, service)
signature = hmac.new(signing_key, string_to_sign.encode(), hashlib.sha256).hexdigest() signature = hmac.new(signing_key, string_to_sign.encode(), hashlib.sha256).hexdigest()
query_with_sig = canonical_query + f"&X-Amz-Signature={signature}" query_with_sig = canonical_query + f"&X-Amz-Signature={signature}"
scheme = request.scheme or "http"
return f"{scheme}://{host}{_canonical_uri(bucket_name, object_key)}?{query_with_sig}" return f"{scheme}://{host}{_canonical_uri(bucket_name, object_key)}?{query_with_sig}"
@@ -1026,6 +1043,7 @@ def bucket_handler(bucket_name: str) -> Response:
try: try:
storage.delete_bucket(bucket_name) storage.delete_bucket(bucket_name)
_bucket_policies().delete_policy(bucket_name) _bucket_policies().delete_policy(bucket_name)
_replication_manager().delete_rule(bucket_name)
except StorageError as exc: except StorageError as exc:
code = "BucketNotEmpty" if "not empty" in str(exc) else "NoSuchBucket" code = "BucketNotEmpty" if "not empty" in str(exc) else "NoSuchBucket"
status = 409 if code == "BucketNotEmpty" else 404 status = 409 if code == "BucketNotEmpty" else 404
@@ -1069,7 +1087,12 @@ def object_handler(bucket_name: str, object_key: str):
_, error = _object_principal("write", bucket_name, object_key) _, error = _object_principal("write", bucket_name, object_key)
if error: if error:
return error return error
stream = request.stream stream = request.stream
content_encoding = request.headers.get("Content-Encoding", "").lower()
if "aws-chunked" in content_encoding:
stream = AwsChunkedDecoder(stream)
metadata = _extract_request_metadata() metadata = _extract_request_metadata()
try: try:
meta = storage.put_object( meta = storage.put_object(
@@ -1089,6 +1112,12 @@ def object_handler(bucket_name: str, object_key: str):
) )
response = Response(status=200) response = Response(status=200)
response.headers["ETag"] = f'"{meta.etag}"' response.headers["ETag"] = f'"{meta.etag}"'
# Trigger replication if not a replication request
user_agent = request.headers.get("User-Agent", "")
if "S3ReplicationAgent" not in user_agent:
_replication_manager().trigger_replication(bucket_name, object_key, action="write")
return response return response
if request.method in {"GET", "HEAD"}: if request.method in {"GET", "HEAD"}:
@@ -1123,6 +1152,12 @@ def object_handler(bucket_name: str, object_key: str):
return error return error
storage.delete_object(bucket_name, object_key) storage.delete_object(bucket_name, object_key)
current_app.logger.info("Object deleted", extra={"bucket": bucket_name, "key": object_key}) current_app.logger.info("Object deleted", extra={"bucket": bucket_name, "key": object_key})
# Trigger replication if not a replication request
user_agent = request.headers.get("User-Agent", "")
if "S3ReplicationAgent" not in user_agent:
_replication_manager().trigger_replication(bucket_name, object_key, action="delete")
return Response(status=204) return Response(status=204)
@@ -1243,3 +1278,79 @@ def head_object(bucket_name: str, object_key: str) -> Response:
return _error_response("NoSuchKey", "Object not found", 404) return _error_response("NoSuchKey", "Object not found", 404)
except IamError as exc: except IamError as exc:
return _error_response("AccessDenied", str(exc), 403) return _error_response("AccessDenied", str(exc), 403)
class AwsChunkedDecoder:
"""Decodes aws-chunked encoded streams."""
def __init__(self, stream):
self.stream = stream
self.buffer = b""
self.chunk_remaining = 0
self.finished = False
def read(self, size=-1):
if self.finished:
return b""
result = b""
while size == -1 or len(result) < size:
if self.chunk_remaining > 0:
to_read = self.chunk_remaining
if size != -1:
to_read = min(to_read, size - len(result))
chunk = self.stream.read(to_read)
if not chunk:
raise IOError("Unexpected EOF in chunk data")
result += chunk
self.chunk_remaining -= len(chunk)
if self.chunk_remaining == 0:
# Read CRLF after chunk data
crlf = self.stream.read(2)
if crlf != b"\r\n":
raise IOError("Malformed chunk: missing CRLF")
else:
# Read chunk size line
line = b""
while True:
char = self.stream.read(1)
if not char:
if not line: # EOF at start of chunk size
self.finished = True
return result
raise IOError("Unexpected EOF in chunk size")
line += char
if line.endswith(b"\r\n"):
break
# Parse chunk size (hex)
try:
line_str = line.decode("ascii").strip()
# Handle chunk-signature extension if present (e.g. "1000;chunk-signature=...")
if ";" in line_str:
line_str = line_str.split(";")[0]
chunk_size = int(line_str, 16)
except ValueError:
raise IOError(f"Invalid chunk size: {line}")
if chunk_size == 0:
self.finished = True
# Read trailers if any (until empty line)
while True:
line = b""
while True:
char = self.stream.read(1)
if not char:
break
line += char
if line.endswith(b"\r\n"):
break
if line == b"\r\n" or not line:
break
return result
self.chunk_remaining = chunk_size
return result

View File

@@ -6,7 +6,9 @@ import uuid
from typing import Any from typing import Any
from urllib.parse import urlparse from urllib.parse import urlparse
import boto3
import requests import requests
from botocore.exceptions import ClientError
from flask import ( from flask import (
Blueprint, Blueprint,
Response, Response,
@@ -38,6 +40,10 @@ def _storage() -> ObjectStorage:
return current_app.extensions["object_storage"] return current_app.extensions["object_storage"]
def _replication_manager() -> ReplicationManager:
return current_app.extensions["replication"]
def _iam(): def _iam():
return current_app.extensions["iam"] return current_app.extensions["iam"]
@@ -494,6 +500,7 @@ def delete_bucket(bucket_name: str):
_authorize_ui(principal, bucket_name, "delete") _authorize_ui(principal, bucket_name, "delete")
_storage().delete_bucket(bucket_name) _storage().delete_bucket(bucket_name)
_bucket_policies().delete_policy(bucket_name) _bucket_policies().delete_policy(bucket_name)
_replication_manager().delete_rule(bucket_name)
flash(f"Bucket '{bucket_name}' removed", "success") flash(f"Bucket '{bucket_name}' removed", "success")
except (StorageError, IamError) as exc: except (StorageError, IamError) as exc:
flash(_friendly_error_message(exc), "danger") flash(_friendly_error_message(exc), "danger")
@@ -512,6 +519,7 @@ def delete_object(bucket_name: str, object_key: str):
flash(f"Permanently deleted '{object_key}' and all versions", "success") flash(f"Permanently deleted '{object_key}' and all versions", "success")
else: else:
_storage().delete_object(bucket_name, object_key) _storage().delete_object(bucket_name, object_key)
_replication_manager().trigger_replication(bucket_name, object_key, action="delete")
flash(f"Deleted '{object_key}'", "success") flash(f"Deleted '{object_key}'", "success")
except (IamError, StorageError) as exc: except (IamError, StorageError) as exc:
flash(_friendly_error_message(exc), "danger") flash(_friendly_error_message(exc), "danger")
@@ -572,6 +580,7 @@ def bulk_delete_objects(bucket_name: str):
storage.purge_object(bucket_name, key) storage.purge_object(bucket_name, key)
else: else:
storage.delete_object(bucket_name, key) storage.delete_object(bucket_name, key)
_replication_manager().trigger_replication(bucket_name, key, action="delete")
deleted.append(key) deleted.append(key)
except StorageError as exc: except StorageError as exc:
errors.append({"key": key, "error": str(exc)}) errors.append({"key": key, "error": str(exc)})
@@ -701,10 +710,21 @@ def object_presign(bucket_name: str, object_key: str):
_authorize_ui(principal, bucket_name, action, object_key=object_key) _authorize_ui(principal, bucket_name, action, object_key=object_key)
except IamError as exc: except IamError as exc:
return jsonify({"error": str(exc)}), 403 return jsonify({"error": str(exc)}), 403
api_base = current_app.config["API_BASE_URL"].rstrip("/")
api_base = current_app.config.get("API_BASE_URL")
if not api_base:
api_base = "http://127.0.0.1:5000"
api_base = api_base.rstrip("/")
url = f"{api_base}/presign/{bucket_name}/{object_key}" url = f"{api_base}/presign/{bucket_name}/{object_key}"
headers = _api_headers()
# Forward the host so the API knows the public URL
headers["X-Forwarded-Host"] = request.host
headers["X-Forwarded-Proto"] = request.scheme
try: try:
response = requests.post(url, headers=_api_headers(), json=payload, timeout=5) response = requests.post(url, headers=headers, json=payload, timeout=5)
except requests.RequestException as exc: except requests.RequestException as exc:
return jsonify({"error": f"API unavailable: {exc}"}), 502 return jsonify({"error": f"API unavailable: {exc}"}), 502
try: try:
@@ -926,6 +946,12 @@ def rotate_iam_secret(access_key: str):
return redirect(url_for("ui.iam_dashboard")) return redirect(url_for("ui.iam_dashboard"))
try: try:
new_secret = _iam().rotate_secret(access_key) new_secret = _iam().rotate_secret(access_key)
# If rotating own key, update session immediately so subsequent API calls (like presign) work
if principal and principal.access_key == access_key:
creds = session.get("credentials", {})
creds["secret_key"] = new_secret
session["credentials"] = creds
session.modified = True
except IamError as exc: except IamError as exc:
if request.accept_mimetypes.accept_json and not request.accept_mimetypes.accept_html: if request.accept_mimetypes.accept_json and not request.accept_mimetypes.accept_html:
return jsonify({"error": str(exc)}), 400 return jsonify({"error": str(exc)}), 400
@@ -1064,6 +1090,73 @@ def create_connection():
return redirect(url_for("ui.connections_dashboard")) return redirect(url_for("ui.connections_dashboard"))
@ui_bp.post("/connections/test")
def test_connection():
principal = _current_principal()
try:
_iam().authorize(principal, None, "iam:list_users")
except IamError:
return jsonify({"status": "error", "message": "Access denied"}), 403
data = request.get_json(silent=True) or request.form
endpoint = data.get("endpoint_url", "").strip()
access_key = data.get("access_key", "").strip()
secret_key = data.get("secret_key", "").strip()
region = data.get("region", "us-east-1").strip()
if not all([endpoint, access_key, secret_key]):
return jsonify({"status": "error", "message": "Missing credentials"}), 400
try:
s3 = boto3.client(
"s3",
endpoint_url=endpoint,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region_name=region,
)
# Try to list buckets to verify credentials and endpoint
s3.list_buckets()
return jsonify({"status": "ok", "message": "Connection successful"})
except Exception as e:
return jsonify({"status": "error", "message": str(e)}), 400
@ui_bp.post("/connections/<connection_id>/update")
def update_connection(connection_id: str):
principal = _current_principal()
try:
_iam().authorize(principal, None, "iam:list_users")
except IamError:
flash("Access denied", "danger")
return redirect(url_for("ui.buckets_overview"))
conn = _connections().get(connection_id)
if not conn:
flash("Connection not found", "danger")
return redirect(url_for("ui.connections_dashboard"))
name = request.form.get("name", "").strip()
endpoint = request.form.get("endpoint_url", "").strip()
access_key = request.form.get("access_key", "").strip()
secret_key = request.form.get("secret_key", "").strip()
region = request.form.get("region", "us-east-1").strip()
if not all([name, endpoint, access_key, secret_key]):
flash("All fields are required", "danger")
return redirect(url_for("ui.connections_dashboard"))
conn.name = name
conn.endpoint_url = endpoint
conn.access_key = access_key
conn.secret_key = secret_key
conn.region = region
_connections().save()
flash(f"Connection '{name}' updated", "success")
return redirect(url_for("ui.connections_dashboard"))
@ui_bp.post("/connections/<connection_id>/delete") @ui_bp.post("/connections/<connection_id>/delete")
def delete_connection(connection_id: str): def delete_connection(connection_id: str):
principal = _current_principal() principal = _current_principal()

View File

@@ -1,7 +1,7 @@
"""Central location for the application version string.""" """Central location for the application version string."""
from __future__ import annotations from __future__ import annotations
APP_VERSION = "0.1.0" APP_VERSION = "0.1.1"
def get_version() -> str: def get_version() -> str:

5
docker-entrypoint.sh Normal file
View File

@@ -0,0 +1,5 @@
#!/bin/sh
set -e
# Run both services using the python runner in production mode
exec python run.py --prod

25
docs.md
View File

@@ -77,12 +77,20 @@ The repo now tracks a human-friendly release string inside `app/version.py` (see
| `SECRET_KEY` | `dev-secret-key` | Flask session key for UI auth. | | `SECRET_KEY` | `dev-secret-key` | Flask session key for UI auth. |
| `IAM_CONFIG` | `<repo>/data/.myfsio.sys/config/iam.json` | Stores users, secrets, and inline policies. | | `IAM_CONFIG` | `<repo>/data/.myfsio.sys/config/iam.json` | Stores users, secrets, and inline policies. |
| `BUCKET_POLICY_PATH` | `<repo>/data/.myfsio.sys/config/bucket_policies.json` | Bucket policy store (auto hot-reload). | | `BUCKET_POLICY_PATH` | `<repo>/data/.myfsio.sys/config/bucket_policies.json` | Bucket policy store (auto hot-reload). |
| `API_BASE_URL` | `http://127.0.0.1:5000` | Used by the UI to hit API endpoints (presign/policy). | | `API_BASE_URL` | `None` | Used by the UI to hit API endpoints (presign/policy). If unset, the UI will auto-detect the host or use `X-Forwarded-*` headers. |
| `AWS_REGION` | `us-east-1` | Region embedded in SigV4 credential scope. | | `AWS_REGION` | `us-east-1` | Region embedded in SigV4 credential scope. |
| `AWS_SERVICE` | `s3` | Service string for SigV4. | | `AWS_SERVICE` | `s3` | Service string for SigV4. |
Set env vars (or pass overrides to `create_app`) to point the servers at custom paths. Set env vars (or pass overrides to `create_app`) to point the servers at custom paths.
### Proxy Configuration
If running behind a reverse proxy (e.g., Nginx, Cloudflare, or a tunnel), ensure the proxy sets the standard forwarding headers:
- `X-Forwarded-Host`
- `X-Forwarded-Proto`
The application automatically trusts these headers to generate correct presigned URLs (e.g., `https://s3.example.com/...` instead of `http://127.0.0.1:5000/...`). Alternatively, you can explicitly set `API_BASE_URL` to your public endpoint.
## 4. Authentication & IAM ## 4. Authentication & IAM
1. On first boot, `data/.myfsio.sys/config/iam.json` is seeded with `localadmin / localadmin` that has wildcard access. 1. On first boot, `data/.myfsio.sys/config/iam.json` is seeded with `localadmin / localadmin` that has wildcard access.
@@ -262,6 +270,21 @@ Now, configure the primary instance to replicate to the target.
aws --endpoint-url http://target-server:5002 s3 ls s3://backup-bucket aws --endpoint-url http://target-server:5002 s3 ls s3://backup-bucket
``` ```
### Bidirectional Replication (Active-Active)
To set up two-way replication (Server A ↔ Server B):
1. Follow the steps above to replicate **A → B**.
2. Repeat the process on Server B to replicate **B → A**:
- Create a connection on Server B pointing to Server A.
- Enable replication on the target bucket on Server B.
**Loop Prevention**: The system automatically detects replication traffic using a custom User-Agent (`S3ReplicationAgent`). This prevents infinite loops where an object replicated from A to B is immediately replicated back to A.
**Deletes**: Deleting an object on one server will propagate the deletion to the other server.
**Note**: Deleting a bucket will automatically remove its associated replication configuration.
## 7. Running Tests ## 7. Running Tests
```bash ```bash

View File

@@ -5,3 +5,4 @@ Flask-WTF>=1.2.1
pytest>=7.4 pytest>=7.4
requests>=2.31 requests>=2.31
boto3>=1.34 boto3>=1.34
waitress>=2.1.2

33
run.py
View File

@@ -18,20 +18,28 @@ def _is_debug_enabled() -> bool:
return os.getenv("FLASK_DEBUG", "0").lower() in ("1", "true", "yes") return os.getenv("FLASK_DEBUG", "0").lower() in ("1", "true", "yes")
def serve_api(port: int) -> None: def serve_api(port: int, prod: bool = False) -> None:
app = create_api_app() app = create_api_app()
debug = _is_debug_enabled() if prod:
if debug: from waitress import serve
warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning) serve(app, host=_server_host(), port=port, ident="MyFSIO")
app.run(host=_server_host(), port=port, debug=debug) else:
debug = _is_debug_enabled()
if debug:
warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning)
app.run(host=_server_host(), port=port, debug=debug)
def serve_ui(port: int) -> None: def serve_ui(port: int, prod: bool = False) -> None:
app = create_ui_app() app = create_ui_app()
debug = _is_debug_enabled() if prod:
if debug: from waitress import serve
warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning) serve(app, host=_server_host(), port=port, ident="MyFSIO")
app.run(host=_server_host(), port=port, debug=debug) else:
debug = _is_debug_enabled()
if debug:
warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning)
app.run(host=_server_host(), port=port, debug=debug)
if __name__ == "__main__": if __name__ == "__main__":
@@ -39,18 +47,19 @@ if __name__ == "__main__":
parser.add_argument("--mode", choices=["api", "ui", "both"], default="both") parser.add_argument("--mode", choices=["api", "ui", "both"], default="both")
parser.add_argument("--api-port", type=int, default=5000) parser.add_argument("--api-port", type=int, default=5000)
parser.add_argument("--ui-port", type=int, default=5100) parser.add_argument("--ui-port", type=int, default=5100)
parser.add_argument("--prod", action="store_true", help="Run in production mode using Waitress")
args = parser.parse_args() args = parser.parse_args()
if args.mode in {"api", "both"}: if args.mode in {"api", "both"}:
print(f"Starting API server on port {args.api_port}...") print(f"Starting API server on port {args.api_port}...")
api_proc = Process(target=serve_api, args=(args.api_port,), daemon=True) api_proc = Process(target=serve_api, args=(args.api_port, args.prod), daemon=True)
api_proc.start() api_proc.start()
else: else:
api_proc = None api_proc = None
if args.mode in {"ui", "both"}: if args.mode in {"ui", "both"}:
print(f"Starting UI server on port {args.ui_port}...") print(f"Starting UI server on port {args.ui_port}...")
serve_ui(args.ui_port) serve_ui(args.ui_port, args.prod)
elif api_proc: elif api_proc:
try: try:
api_proc.join() api_proc.join()

View File

@@ -408,16 +408,16 @@
</dl> </dl>
</div> </div>
<form method="POST" action="{{ url_for('ui.update_bucket_replication', bucket_name=bucket_name) }}" onsubmit="return confirm('Are you sure you want to disable replication?');"> <button type="button" class="btn btn-danger" data-bs-toggle="modal" data-bs-target="#disableReplicationModal">
<input type="hidden" name="action" value="delete"> Disable Replication
<button type="submit" class="btn btn-danger">Disable Replication</button> </button>
</form>
{% else %} {% else %}
<p class="text-muted">Replication allows you to automatically copy new objects from this bucket to a bucket in another S3-compatible service.</p> <p class="text-muted">Replication allows you to automatically copy new objects from this bucket to a bucket in another S3-compatible service.</p>
{% if connections %} {% if connections %}
<form method="POST" action="{{ url_for('ui.update_bucket_replication', bucket_name=bucket_name) }}"> <form method="POST" action="{{ url_for('ui.update_bucket_replication', bucket_name=bucket_name) }}">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="action" value="create"> <input type="hidden" name="action" value="create">
<div class="mb-3"> <div class="mb-3">
@@ -434,7 +434,7 @@
<div class="mb-3"> <div class="mb-3">
<label for="target_bucket" class="form-label">Target Bucket Name</label> <label for="target_bucket" class="form-label">Target Bucket Name</label>
<input type="text" class="form-control" id="target_bucket" name="target_bucket" required placeholder="e.g. my-backup-bucket"> <input type="text" class="form-control" id="target_bucket" name="target_bucket" required placeholder="e.g. my-backup-bucket">
<div class="form-text">The bucket on the remote service must already exist.</div> <div class="form-text">If the target bucket does not exist, it will be created automatically.</div>
</div> </div>
<button type="submit" class="btn btn-primary">Enable Replication</button> <button type="submit" class="btn btn-primary">Enable Replication</button>
@@ -708,6 +708,30 @@
</div> </div>
</div> </div>
</div> </div>
<!-- Disable Replication Modal -->
<div class="modal fade" id="disableReplicationModal" tabindex="-1" aria-hidden="true">
<div class="modal-dialog modal-dialog-centered">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title">Disable Replication</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
<p>Are you sure you want to disable replication for this bucket?</p>
<p class="text-muted small">Existing objects in the target bucket will remain, but new uploads will no longer be copied.</p>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
<form method="POST" action="{{ url_for('ui.update_bucket_replication', bucket_name=bucket_name) }}">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<input type="hidden" name="action" value="delete">
<button type="submit" class="btn btn-danger">Disable Replication</button>
</form>
</div>
</div>
</div>
</div>
{% endblock %} {% endblock %}
{% block extra_scripts %} {% block extra_scripts %}

View File

@@ -12,12 +12,13 @@
<div class="row"> <div class="row">
<div class="col-md-4"> <div class="col-md-4">
<div class="card"> <div class="card shadow-sm">
<div class="card-header"> <div class="card-header fw-semibold">
Add New Connection Add New Connection
</div> </div>
<div class="card-body"> <div class="card-body">
<form method="POST" action="{{ url_for('ui.create_connection') }}"> <form method="POST" action="{{ url_for('ui.create_connection') }}" id="createConnectionForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<div class="mb-3"> <div class="mb-3">
<label for="name" class="form-label">Name</label> <label for="name" class="form-label">Name</label>
<input type="text" class="form-control" id="name" name="name" required placeholder="e.g. Production Backup"> <input type="text" class="form-control" id="name" name="name" required placeholder="e.g. Production Backup">
@@ -36,43 +37,69 @@
</div> </div>
<div class="mb-3"> <div class="mb-3">
<label for="secret_key" class="form-label">Secret Key</label> <label for="secret_key" class="form-label">Secret Key</label>
<input type="password" class="form-control" id="secret_key" name="secret_key" required> <div class="input-group">
<input type="password" class="form-control" id="secret_key" name="secret_key" required>
<button class="btn btn-outline-secondary" type="button" onclick="togglePassword('secret_key')">
<i class="bi bi-eye"></i> Show
</button>
</div>
</div> </div>
<button type="submit" class="btn btn-primary">Add Connection</button> <div class="d-grid gap-2">
<button type="button" class="btn btn-outline-secondary" id="testConnectionBtn">Test Connection</button>
<button type="submit" class="btn btn-primary">Add Connection</button>
</div>
<div id="testResult" class="mt-2"></div>
</form> </form>
</div> </div>
</div> </div>
</div> </div>
<div class="col-md-8"> <div class="col-md-8">
<div class="card"> <div class="card shadow-sm">
<div class="card-header"> <div class="card-header fw-semibold">
Existing Connections Existing Connections
</div> </div>
<div class="card-body"> <div class="card-body">
{% if connections %} {% if connections %}
<div class="table-responsive"> <div class="table-responsive">
<table class="table table-hover"> <table class="table table-hover align-middle">
<thead> <thead>
<tr> <tr>
<th>Name</th> <th>Name</th>
<th>Endpoint</th> <th>Endpoint</th>
<th>Region</th> <th>Region</th>
<th>Access Key</th> <th>Access Key</th>
<th>Actions</th> <th class="text-end">Actions</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
{% for conn in connections %} {% for conn in connections %}
<tr> <tr>
<td>{{ conn.name }}</td> <td class="fw-medium">{{ conn.name }}</td>
<td>{{ conn.endpoint_url }}</td> <td class="small text-muted">{{ conn.endpoint_url }}</td>
<td>{{ conn.region }}</td> <td><span class="badge bg-light text-dark border">{{ conn.region }}</span></td>
<td><code>{{ conn.access_key }}</code></td> <td><code class="small">{{ conn.access_key }}</code></td>
<td> <td class="text-end">
<form method="POST" action="{{ url_for('ui.delete_connection', connection_id=conn.id) }}" onsubmit="return confirm('Are you sure?');" style="display: inline;"> <div class="btn-group">
<button type="submit" class="btn btn-sm btn-danger">Delete</button> <button type="button" class="btn btn-sm btn-outline-primary"
</form> data-bs-toggle="modal"
data-bs-target="#editConnectionModal"
data-id="{{ conn.id }}"
data-name="{{ conn.name }}"
data-endpoint="{{ conn.endpoint_url }}"
data-region="{{ conn.region }}"
data-access="{{ conn.access_key }}"
data-secret="{{ conn.secret_key }}">
Edit
</button>
<button type="button" class="btn btn-sm btn-outline-danger"
data-bs-toggle="modal"
data-bs-target="#deleteConnectionModal"
data-id="{{ conn.id }}"
data-name="{{ conn.name }}">
Delete
</button>
</div>
</td> </td>
</tr> </tr>
{% endfor %} {% endfor %}
@@ -80,10 +107,164 @@
</table> </table>
</div> </div>
{% else %} {% else %}
<p class="text-muted text-center my-4">No remote connections configured.</p> <div class="text-center py-5 text-muted">
<svg xmlns="http://www.w3.org/2000/svg" width="48" height="48" fill="currentColor" class="bi bi-hdd-network mb-3" viewBox="0 0 16 16">
<path d="M4.5 5a.5.5 0 1 0 0-1 .5.5 0 0 0 0 1zM3 4.5a.5.5 0 1 1-1 0 .5.5 0 0 1 1 0z"/>
<path d="M0 4a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v1a2 2 0 0 1-2 2H8.5v3a1.5 1.5 0 0 1 1.5 1.5v3.375a1.125 1.125 0 0 1-1.125 1.125h-1.75a1.125 1.125 0 0 1-1.125-1.125V11.5A1.5 1.5 0 0 1 7.5 10V7H2a2 2 0 0 1-2-2V4zm1 0v1a1 1 0 0 0 1 1h12a1 1 0 0 0 1-1V4a1 1 0 0 0-1-1H2a1 1 0 0 0-1 1zm6 5v1.5a.5.5 0 0 0 .5.5h1.75a.5.5 0 0 0 .5-.5V10a.5.5 0 0 0-.5-.5H7.5a.5.5 0 0 0-.5.5z"/>
</svg>
<p>No remote connections configured.</p>
</div>
{% endif %} {% endif %}
</div> </div>
</div> </div>
</div> </div>
</div> </div>
<!-- Edit Connection Modal -->
<div class="modal fade" id="editConnectionModal" tabindex="-1" aria-hidden="true">
<div class="modal-dialog">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title">Edit Connection</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<form method="POST" id="editConnectionForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<div class="modal-body">
<div class="mb-3">
<label for="edit_name" class="form-label">Name</label>
<input type="text" class="form-control" id="edit_name" name="name" required>
</div>
<div class="mb-3">
<label for="edit_endpoint_url" class="form-label">Endpoint URL</label>
<input type="url" class="form-control" id="edit_endpoint_url" name="endpoint_url" required>
</div>
<div class="mb-3">
<label for="edit_region" class="form-label">Region</label>
<input type="text" class="form-control" id="edit_region" name="region" required>
</div>
<div class="mb-3">
<label for="edit_access_key" class="form-label">Access Key</label>
<input type="text" class="form-control" id="edit_access_key" name="access_key" required>
</div>
<div class="mb-3">
<label for="edit_secret_key" class="form-label">Secret Key</label>
<div class="input-group">
<input type="password" class="form-control" id="edit_secret_key" name="secret_key" required>
<button class="btn btn-outline-secondary" type="button" onclick="togglePassword('edit_secret_key')">
Show
</button>
</div>
</div>
<div id="editTestResult" class="mt-2"></div>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-outline-secondary" id="editTestConnectionBtn">Test Connection</button>
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
<button type="submit" class="btn btn-primary">Save Changes</button>
</div>
</form>
</div>
</div>
</div>
<!-- Delete Connection Modal -->
<div class="modal fade" id="deleteConnectionModal" tabindex="-1" aria-hidden="true">
<div class="modal-dialog modal-dialog-centered">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title">Delete Connection</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
<p>Are you sure you want to delete the connection <strong id="deleteConnectionName"></strong>?</p>
<p class="text-muted small">This will stop any replication rules using this connection.</p>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
<form method="POST" id="deleteConnectionForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<button type="submit" class="btn btn-danger">Delete</button>
</form>
</div>
</div>
</div>
</div>
<script>
function togglePassword(id) {
const input = document.getElementById(id);
if (input.type === "password") {
input.type = "text";
} else {
input.type = "password";
}
}
// Test Connection Logic
async function testConnection(formId, resultId) {
const form = document.getElementById(formId);
const resultDiv = document.getElementById(resultId);
const formData = new FormData(form);
const data = Object.fromEntries(formData.entries());
resultDiv.innerHTML = '<div class="text-info"><span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span> Testing...</div>';
try {
const response = await fetch("{{ url_for('ui.test_connection') }}", {
method: "POST",
headers: {
"Content-Type": "application/json",
"X-CSRFToken": "{{ csrf_token() }}"
},
body: JSON.stringify(data)
});
const result = await response.json();
if (response.ok) {
resultDiv.innerHTML = `<div class="text-success"><i class="bi bi-check-circle"></i> ${result.message}</div>`;
} else {
resultDiv.innerHTML = `<div class="text-danger"><i class="bi bi-exclamation-circle"></i> ${result.message}</div>`;
}
} catch (error) {
resultDiv.innerHTML = `<div class="text-danger"><i class="bi bi-exclamation-circle"></i> Connection failed</div>`;
}
}
document.getElementById('testConnectionBtn').addEventListener('click', () => {
testConnection('createConnectionForm', 'testResult');
});
document.getElementById('editTestConnectionBtn').addEventListener('click', () => {
testConnection('editConnectionForm', 'editTestResult');
});
// Modal Event Listeners
const editModal = document.getElementById('editConnectionModal');
editModal.addEventListener('show.bs.modal', event => {
const button = event.relatedTarget;
const id = button.getAttribute('data-id');
document.getElementById('edit_name').value = button.getAttribute('data-name');
document.getElementById('edit_endpoint_url').value = button.getAttribute('data-endpoint');
document.getElementById('edit_region').value = button.getAttribute('data-region');
document.getElementById('edit_access_key').value = button.getAttribute('data-access');
document.getElementById('edit_secret_key').value = button.getAttribute('data-secret');
document.getElementById('editTestResult').innerHTML = '';
const form = document.getElementById('editConnectionForm');
form.action = "{{ url_for('ui.update_connection', connection_id='CONN_ID') }}".replace('CONN_ID', id);
});
const deleteModal = document.getElementById('deleteConnectionModal');
deleteModal.addEventListener('show.bs.modal', event => {
const button = event.relatedTarget;
const id = button.getAttribute('data-id');
const name = button.getAttribute('data-name');
document.getElementById('deleteConnectionName').textContent = name;
const form = document.getElementById('deleteConnectionForm');
form.action = "{{ url_for('ui.delete_connection', connection_id='CONN_ID') }}".replace('CONN_ID', id);
});
</script>
{% endblock %} {% endblock %}

View File

@@ -290,6 +290,18 @@ s3.complete_multipart_upload(
</div> </div>
</div> </div>
</div> </div>
<h3 class="h6 text-uppercase text-muted mt-4">Bidirectional Replication (Active-Active)</h3>
<p class="small text-muted">To set up two-way replication (Server A ↔ Server B):</p>
<ol class="docs-steps mb-3">
<li>Follow the steps above to replicate <strong>A → B</strong>.</li>
<li>Repeat the process on Server B to replicate <strong>B → A</strong> (create a connection to A, enable rule).</li>
</ol>
<p class="small text-muted mb-0">
<strong>Loop Prevention:</strong> The system automatically detects replication traffic using a custom User-Agent (<code>S3ReplicationAgent</code>). This prevents infinite loops where an object replicated from A to B is immediately replicated back to A.
<br>
<strong>Deletes:</strong> Deleting an object on one server will propagate the deletion to the other server.
</p>
</div> </div>
</article> </article>
<article id="troubleshooting" class="card shadow-sm docs-section"> <article id="troubleshooting" class="card shadow-sm docs-section">
@@ -330,8 +342,8 @@ s3.complete_multipart_upload(
</tr> </tr>
<tr> <tr>
<td>Requests hit the wrong host</td> <td>Requests hit the wrong host</td>
<td><code>API_BASE_URL</code> not updated after tunneling/forwarding</td> <td>Proxy headers missing or <code>API_BASE_URL</code> incorrect</td>
<td>Set <code>API_BASE_URL</code> in your shell or <code>.env</code> to match the published host.</td> <td>Ensure your proxy sends <code>X-Forwarded-Host</code>/<code>Proto</code> headers, or explicitly set <code>API_BASE_URL</code> to your public domain.</td>
</tr> </tr>
</tbody> </tbody>
</table> </table>

View File

@@ -286,9 +286,6 @@
</div> </div>
<div class="modal-body" id="rotateSecretConfirm"> <div class="modal-body" id="rotateSecretConfirm">
<p>Are you sure you want to rotate the secret key for <strong id="rotateUserLabel"></strong>?</p> <p>Are you sure you want to rotate the secret key for <strong id="rotateUserLabel"></strong>?</p>
<div id="rotateSelfWarning" class="alert alert-warning d-none">
<strong>Warning:</strong> You are rotating your own secret key. You will need to sign in again with the new key.
</div>
<div class="alert alert-warning mb-0"> <div class="alert alert-warning mb-0">
The old secret key will stop working immediately. Any applications using it must be updated. The old secret key will stop working immediately. Any applications using it must be updated.
</div> </div>
@@ -474,7 +471,6 @@
const rotateSecretResult = document.getElementById('rotateSecretResult'); const rotateSecretResult = document.getElementById('rotateSecretResult');
const newSecretKeyInput = document.getElementById('newSecretKey'); const newSecretKeyInput = document.getElementById('newSecretKey');
const copyNewSecretBtn = document.getElementById('copyNewSecret'); const copyNewSecretBtn = document.getElementById('copyNewSecret');
const rotateSelfWarning = document.getElementById('rotateSelfWarning');
let currentRotateKey = null; let currentRotateKey = null;
document.querySelectorAll('[data-rotate-user]').forEach(btn => { document.querySelectorAll('[data-rotate-user]').forEach(btn => {
@@ -482,12 +478,6 @@
currentRotateKey = btn.dataset.rotateUser; currentRotateKey = btn.dataset.rotateUser;
rotateUserLabel.textContent = currentRotateKey; rotateUserLabel.textContent = currentRotateKey;
if (currentRotateKey === currentUserKey) {
rotateSelfWarning.classList.remove('d-none');
} else {
rotateSelfWarning.classList.add('d-none');
}
// Reset Modal State // Reset Modal State
rotateSecretConfirm.classList.remove('d-none'); rotateSecretConfirm.classList.remove('d-none');
rotateSecretResult.classList.add('d-none'); rotateSecretResult.classList.add('d-none');