Compare commits
21 Commits
f400cedf02
...
c6cbe822e1
| Author | SHA1 | Date | |
|---|---|---|---|
| c6cbe822e1 | |||
| dddab6dbbc | |||
| 015c9cb52d | |||
| c8b1c33118 | |||
| ebef3dfa57 | |||
| 1116353d0f | |||
| e4b92a32a1 | |||
| 57c40dcdcc | |||
| 7d1735a59f | |||
| 9064f9d60e | |||
| 36c08b0ac1 | |||
| ec5d52f208 | |||
| 96de6164d1 | |||
| 8c00d7bd4b | |||
| a32d9dbd77 | |||
| fe3eacd2be | |||
| 471cf5a305 | |||
| 840fd176d3 | |||
| 5350d04ba5 | |||
| f2daa8a8a3 | |||
| e287b59645 |
13
.dockerignore
Normal file
13
.dockerignore
Normal file
@@ -0,0 +1,13 @@
|
||||
.git
|
||||
.gitignore
|
||||
.venv
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
.pytest_cache
|
||||
.coverage
|
||||
htmlcov
|
||||
logs
|
||||
data
|
||||
tmp
|
||||
11
Dockerfile
11
Dockerfile
@@ -16,9 +16,14 @@ RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
# Drop privileges
|
||||
RUN useradd -m -u 1000 myfsio \
|
||||
# Make entrypoint executable
|
||||
RUN chmod +x docker-entrypoint.sh
|
||||
|
||||
# Create data directory and set permissions
|
||||
RUN mkdir -p /app/data \
|
||||
&& useradd -m -u 1000 myfsio \
|
||||
&& chown -R myfsio:myfsio /app
|
||||
|
||||
USER myfsio
|
||||
|
||||
EXPOSE 5000 5100
|
||||
@@ -29,4 +34,4 @@ ENV APP_HOST=0.0.0.0 \
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD python -c "import requests; requests.get('http://localhost:5000/healthz', timeout=2)"
|
||||
|
||||
CMD ["python", "run.py", "--mode", "both"]
|
||||
CMD ["./docker-entrypoint.sh"]
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from logging.handlers import RotatingFileHandler
|
||||
@@ -12,6 +13,7 @@ from typing import Any, Dict, Optional
|
||||
from flask import Flask, g, has_request_context, redirect, render_template, request, url_for
|
||||
from flask_cors import CORS
|
||||
from flask_wtf.csrf import CSRFError
|
||||
from werkzeug.middleware.proxy_fix import ProxyFix
|
||||
|
||||
from .bucket_policies import BucketPolicyStore
|
||||
from .config import AppConfig
|
||||
@@ -47,6 +49,9 @@ def create_app(
|
||||
if app.config.get("TESTING"):
|
||||
app.config.setdefault("WTF_CSRF_ENABLED", False)
|
||||
|
||||
# Trust X-Forwarded-* headers from proxies
|
||||
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1)
|
||||
|
||||
_configure_cors(app)
|
||||
_configure_logging(app)
|
||||
|
||||
@@ -167,23 +172,33 @@ class _RequestContextFilter(logging.Filter):
|
||||
|
||||
|
||||
def _configure_logging(app: Flask) -> None:
|
||||
log_file = Path(app.config["LOG_FILE"])
|
||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
handler = RotatingFileHandler(
|
||||
log_file,
|
||||
maxBytes=int(app.config.get("LOG_MAX_BYTES", 5 * 1024 * 1024)),
|
||||
backupCount=int(app.config.get("LOG_BACKUP_COUNT", 3)),
|
||||
encoding="utf-8",
|
||||
)
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s | %(levelname)s | %(request_id)s | %(method)s %(path)s | %(message)s"
|
||||
)
|
||||
handler.setFormatter(formatter)
|
||||
handler.addFilter(_RequestContextFilter())
|
||||
|
||||
# Stream Handler (stdout) - Primary for Docker
|
||||
stream_handler = logging.StreamHandler(sys.stdout)
|
||||
stream_handler.setFormatter(formatter)
|
||||
stream_handler.addFilter(_RequestContextFilter())
|
||||
|
||||
logger = app.logger
|
||||
logger.handlers.clear()
|
||||
logger.addHandler(handler)
|
||||
logger.addHandler(stream_handler)
|
||||
|
||||
# File Handler (optional, if configured)
|
||||
if app.config.get("LOG_TO_FILE"):
|
||||
log_file = Path(app.config["LOG_FILE"])
|
||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_handler = RotatingFileHandler(
|
||||
log_file,
|
||||
maxBytes=int(app.config.get("LOG_MAX_BYTES", 5 * 1024 * 1024)),
|
||||
backupCount=int(app.config.get("LOG_BACKUP_COUNT", 3)),
|
||||
encoding="utf-8",
|
||||
)
|
||||
file_handler.setFormatter(formatter)
|
||||
file_handler.addFilter(_RequestContextFilter())
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
logger.setLevel(getattr(logging, app.config.get("LOG_LEVEL", "INFO"), logging.INFO))
|
||||
|
||||
@app.before_request
|
||||
@@ -211,5 +226,4 @@ def _configure_logging(app: Flask) -> None:
|
||||
},
|
||||
)
|
||||
response.headers["X-Request-Duration-ms"] = f"{duration_ms:.2f}"
|
||||
response.headers["Server"] = "MyFISO"
|
||||
return response
|
||||
|
||||
@@ -39,7 +39,7 @@ class AppConfig:
|
||||
secret_key: str
|
||||
iam_config_path: Path
|
||||
bucket_policy_path: Path
|
||||
api_base_url: str
|
||||
api_base_url: Optional[str]
|
||||
aws_region: str
|
||||
aws_service: str
|
||||
ui_enforce_bucket_policies: bool
|
||||
@@ -78,11 +78,25 @@ class AppConfig:
|
||||
multipart_min_part_size = int(_get("MULTIPART_MIN_PART_SIZE", 5 * 1024 * 1024))
|
||||
default_secret = "dev-secret-key"
|
||||
secret_key = str(_get("SECRET_KEY", default_secret))
|
||||
|
||||
# If using default/missing secret, try to load/persist a generated one from disk
|
||||
# This ensures consistency across Gunicorn workers
|
||||
if not secret_key or secret_key == default_secret:
|
||||
generated = secrets.token_urlsafe(32)
|
||||
if secret_key == default_secret:
|
||||
warnings.warn("Using insecure default SECRET_KEY. A random value has been generated; set SECRET_KEY for production", RuntimeWarning)
|
||||
secret_key = generated
|
||||
secret_file = storage_root / ".myfsio.sys" / "config" / ".secret"
|
||||
if secret_file.exists():
|
||||
secret_key = secret_file.read_text().strip()
|
||||
else:
|
||||
generated = secrets.token_urlsafe(32)
|
||||
if secret_key == default_secret:
|
||||
warnings.warn("Using insecure default SECRET_KEY. A random value has been generated and persisted; set SECRET_KEY for production", RuntimeWarning)
|
||||
try:
|
||||
secret_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
secret_file.write_text(generated)
|
||||
secret_key = generated
|
||||
except OSError:
|
||||
# Fallback if we can't write to disk (e.g. read-only fs)
|
||||
secret_key = generated
|
||||
|
||||
iam_env_override = "IAM_CONFIG" in overrides or "IAM_CONFIG" in os.environ
|
||||
bucket_policy_override = "BUCKET_POLICY_PATH" in overrides or "BUCKET_POLICY_PATH" in os.environ
|
||||
|
||||
@@ -100,7 +114,10 @@ class AppConfig:
|
||||
bucket_policy_path,
|
||||
legacy_path=None if bucket_policy_override else PROJECT_ROOT / "data" / "bucket_policies.json",
|
||||
)
|
||||
api_base_url = str(_get("API_BASE_URL", "http://127.0.0.1:5000"))
|
||||
api_base_url = _get("API_BASE_URL", None)
|
||||
if api_base_url:
|
||||
api_base_url = str(api_base_url)
|
||||
|
||||
aws_region = str(_get("AWS_REGION", "us-east-1"))
|
||||
aws_service = str(_get("AWS_SERVICE", "s3"))
|
||||
enforce_ui_policies = str(_get("UI_ENFORCE_BUCKET_POLICIES", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
13
app/iam.py
13
app/iam.py
@@ -77,10 +77,20 @@ class IamService:
|
||||
self._users: Dict[str, Dict[str, Any]] = {}
|
||||
self._raw_config: Dict[str, Any] = {}
|
||||
self._failed_attempts: Dict[str, Deque[datetime]] = {}
|
||||
self._last_load_time = 0.0
|
||||
self._load()
|
||||
|
||||
def _maybe_reload(self) -> None:
|
||||
"""Reload configuration if the file has changed on disk."""
|
||||
try:
|
||||
if self.config_path.stat().st_mtime > self._last_load_time:
|
||||
self._load()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# ---------------------- authz helpers ----------------------
|
||||
def authenticate(self, access_key: str, secret_key: str) -> Principal:
|
||||
self._maybe_reload()
|
||||
access_key = (access_key or "").strip()
|
||||
secret_key = (secret_key or "").strip()
|
||||
if not access_key or not secret_key:
|
||||
@@ -135,12 +145,14 @@ class IamService:
|
||||
return int(max(0, self.auth_lockout_window.total_seconds() - elapsed))
|
||||
|
||||
def principal_for_key(self, access_key: str) -> Principal:
|
||||
self._maybe_reload()
|
||||
record = self._users.get(access_key)
|
||||
if not record:
|
||||
raise IamError("Unknown access key")
|
||||
return self._build_principal(access_key, record)
|
||||
|
||||
def secret_for_key(self, access_key: str) -> str:
|
||||
self._maybe_reload()
|
||||
record = self._users.get(access_key)
|
||||
if not record:
|
||||
raise IamError("Unknown access key")
|
||||
@@ -245,6 +257,7 @@ class IamService:
|
||||
# ---------------------- config helpers ----------------------
|
||||
def _load(self) -> None:
|
||||
try:
|
||||
self._last_load_time = self.config_path.stat().st_mtime
|
||||
content = self.config_path.read_text(encoding='utf-8')
|
||||
raw = json.loads(content)
|
||||
except FileNotFoundError:
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import mimetypes
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
@@ -9,13 +10,17 @@ from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from boto3.exceptions import S3UploadFailedError
|
||||
|
||||
from .connections import ConnectionStore, RemoteConnection
|
||||
from .storage import ObjectStorage
|
||||
from .storage import ObjectStorage, StorageError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
REPLICATION_USER_AGENT = "S3ReplicationAgent/1.0"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReplicationRule:
|
||||
@@ -66,7 +71,26 @@ class ReplicationManager:
|
||||
del self._rules[bucket_name]
|
||||
self.save_rules()
|
||||
|
||||
def trigger_replication(self, bucket_name: str, object_key: str) -> None:
|
||||
def create_remote_bucket(self, connection_id: str, bucket_name: str) -> None:
|
||||
"""Create a bucket on the remote connection."""
|
||||
connection = self.connections.get(connection_id)
|
||||
if not connection:
|
||||
raise ValueError(f"Connection {connection_id} not found")
|
||||
|
||||
try:
|
||||
s3 = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=connection.endpoint_url,
|
||||
aws_access_key_id=connection.access_key,
|
||||
aws_secret_access_key=connection.secret_key,
|
||||
region_name=connection.region,
|
||||
)
|
||||
s3.create_bucket(Bucket=bucket_name)
|
||||
except ClientError as e:
|
||||
logger.error(f"Failed to create remote bucket {bucket_name}: {e}")
|
||||
raise
|
||||
|
||||
def trigger_replication(self, bucket_name: str, object_key: str, action: str = "write") -> None:
|
||||
rule = self.get_rule(bucket_name)
|
||||
if not rule or not rule.enabled:
|
||||
return
|
||||
@@ -76,42 +100,93 @@ class ReplicationManager:
|
||||
logger.warning(f"Replication skipped for {bucket_name}/{object_key}: Connection {rule.target_connection_id} not found")
|
||||
return
|
||||
|
||||
self._executor.submit(self._replicate_task, bucket_name, object_key, rule, connection)
|
||||
self._executor.submit(self._replicate_task, bucket_name, object_key, rule, connection, action)
|
||||
|
||||
def _replicate_task(self, bucket_name: str, object_key: str, rule: ReplicationRule, conn: RemoteConnection) -> None:
|
||||
def _replicate_task(self, bucket_name: str, object_key: str, rule: ReplicationRule, conn: RemoteConnection, action: str) -> None:
|
||||
try:
|
||||
# 1. Get local file path
|
||||
# Note: We are accessing internal storage structure here.
|
||||
# Ideally storage.py should expose a 'get_file_path' or we read the stream.
|
||||
# For efficiency, we'll try to read the file directly if we can, or use storage.get_object
|
||||
|
||||
# Using boto3 to upload
|
||||
config = Config(user_agent_extra=REPLICATION_USER_AGENT)
|
||||
s3 = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=conn.endpoint_url,
|
||||
aws_access_key_id=conn.access_key,
|
||||
aws_secret_access_key=conn.secret_key,
|
||||
region_name=conn.region,
|
||||
config=config,
|
||||
)
|
||||
|
||||
if action == "delete":
|
||||
try:
|
||||
s3.delete_object(Bucket=rule.target_bucket, Key=object_key)
|
||||
logger.info(f"Replicated DELETE {bucket_name}/{object_key} to {conn.name} ({rule.target_bucket})")
|
||||
except ClientError as e:
|
||||
logger.error(f"Replication DELETE failed for {bucket_name}/{object_key}: {e}")
|
||||
return
|
||||
|
||||
# 1. Get local file path
|
||||
# Note: We are accessing internal storage structure here.
|
||||
# Ideally storage.py should expose a 'get_file_path' or we read the stream.
|
||||
# For efficiency, we'll try to read the file directly if we can, or use storage.get_object
|
||||
|
||||
# We need the file content.
|
||||
# Since ObjectStorage is filesystem based, let's get the stream.
|
||||
# We need to be careful about closing it.
|
||||
meta = self.storage.get_object_meta(bucket_name, object_key)
|
||||
if not meta:
|
||||
try:
|
||||
path = self.storage.get_object_path(bucket_name, object_key)
|
||||
except StorageError:
|
||||
return
|
||||
|
||||
with self.storage.open_object(bucket_name, object_key) as f:
|
||||
extra_args = {}
|
||||
if meta.metadata:
|
||||
extra_args["Metadata"] = meta.metadata
|
||||
|
||||
s3.upload_fileobj(
|
||||
f,
|
||||
rule.target_bucket,
|
||||
object_key,
|
||||
ExtraArgs=extra_args
|
||||
)
|
||||
metadata = self.storage.get_object_metadata(bucket_name, object_key)
|
||||
|
||||
extra_args = {}
|
||||
if metadata:
|
||||
extra_args["Metadata"] = metadata
|
||||
|
||||
# Guess content type to prevent corruption/wrong handling
|
||||
content_type, _ = mimetypes.guess_type(path)
|
||||
file_size = path.stat().st_size
|
||||
|
||||
logger.info(f"Replicating {bucket_name}/{object_key}: Size={file_size}, ContentType={content_type}")
|
||||
|
||||
try:
|
||||
with path.open("rb") as f:
|
||||
s3.put_object(
|
||||
Bucket=rule.target_bucket,
|
||||
Key=object_key,
|
||||
Body=f,
|
||||
ContentLength=file_size,
|
||||
ContentType=content_type or "application/octet-stream",
|
||||
Metadata=metadata or {}
|
||||
)
|
||||
except (ClientError, S3UploadFailedError) as e:
|
||||
# Check if it's a NoSuchBucket error (either direct or wrapped)
|
||||
is_no_bucket = False
|
||||
if isinstance(e, ClientError):
|
||||
if e.response['Error']['Code'] == 'NoSuchBucket':
|
||||
is_no_bucket = True
|
||||
elif isinstance(e, S3UploadFailedError):
|
||||
if "NoSuchBucket" in str(e):
|
||||
is_no_bucket = True
|
||||
|
||||
if is_no_bucket:
|
||||
logger.info(f"Target bucket {rule.target_bucket} not found. Attempting to create it.")
|
||||
try:
|
||||
s3.create_bucket(Bucket=rule.target_bucket)
|
||||
# Retry upload
|
||||
with path.open("rb") as f:
|
||||
s3.put_object(
|
||||
Bucket=rule.target_bucket,
|
||||
Key=object_key,
|
||||
Body=f,
|
||||
ContentLength=file_size,
|
||||
ContentType=content_type or "application/octet-stream",
|
||||
Metadata=metadata or {}
|
||||
)
|
||||
except Exception as create_err:
|
||||
logger.error(f"Failed to create target bucket {rule.target_bucket}: {create_err}")
|
||||
raise e # Raise original error
|
||||
else:
|
||||
raise e
|
||||
|
||||
logger.info(f"Replicated {bucket_name}/{object_key} to {conn.name} ({rule.target_bucket})")
|
||||
|
||||
|
||||
119
app/s3_api.py
119
app/s3_api.py
@@ -8,7 +8,7 @@ import re
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict
|
||||
from urllib.parse import quote, urlencode
|
||||
from urllib.parse import quote, urlencode, urlparse
|
||||
from xml.etree.ElementTree import Element, SubElement, tostring, fromstring, ParseError
|
||||
|
||||
from flask import Blueprint, Response, current_app, jsonify, request
|
||||
@@ -17,6 +17,7 @@ from werkzeug.http import http_date
|
||||
from .bucket_policies import BucketPolicyStore
|
||||
from .extensions import limiter
|
||||
from .iam import IamError, Principal
|
||||
from .replication import ReplicationManager
|
||||
from .storage import ObjectStorage, StorageError
|
||||
|
||||
s3_api_bp = Blueprint("s3_api", __name__)
|
||||
@@ -31,6 +32,9 @@ def _iam():
|
||||
return current_app.extensions["iam"]
|
||||
|
||||
|
||||
def _replication_manager() -> ReplicationManager:
|
||||
return current_app.extensions["replication"]
|
||||
|
||||
|
||||
def _bucket_policies() -> BucketPolicyStore:
|
||||
store: BucketPolicyStore = current_app.extensions["bucket_policies"]
|
||||
@@ -405,7 +409,11 @@ def _canonical_headers_from_request(headers: list[str]) -> str:
|
||||
lines = []
|
||||
for header in headers:
|
||||
if header == "host":
|
||||
value = request.host
|
||||
api_base = current_app.config.get("API_BASE_URL")
|
||||
if api_base:
|
||||
value = urlparse(api_base).netloc
|
||||
else:
|
||||
value = request.host
|
||||
else:
|
||||
value = request.headers.get(header, "")
|
||||
canonical_value = " ".join(value.strip().split()) if value else ""
|
||||
@@ -468,7 +476,17 @@ def _generate_presigned_url(
|
||||
"X-Amz-Content-Sha256": "UNSIGNED-PAYLOAD",
|
||||
}
|
||||
canonical_query = _encode_query_params(query_params)
|
||||
host = request.host
|
||||
|
||||
# Determine host and scheme from config or request
|
||||
api_base = current_app.config.get("API_BASE_URL")
|
||||
if api_base:
|
||||
parsed = urlparse(api_base)
|
||||
host = parsed.netloc
|
||||
scheme = parsed.scheme
|
||||
else:
|
||||
host = request.headers.get("X-Forwarded-Host", request.host)
|
||||
scheme = request.headers.get("X-Forwarded-Proto", request.scheme or "http")
|
||||
|
||||
canonical_headers = f"host:{host}\n"
|
||||
canonical_request = "\n".join(
|
||||
[
|
||||
@@ -492,7 +510,6 @@ def _generate_presigned_url(
|
||||
signing_key = _derive_signing_key(secret_key, date_stamp, region, service)
|
||||
signature = hmac.new(signing_key, string_to_sign.encode(), hashlib.sha256).hexdigest()
|
||||
query_with_sig = canonical_query + f"&X-Amz-Signature={signature}"
|
||||
scheme = request.scheme or "http"
|
||||
return f"{scheme}://{host}{_canonical_uri(bucket_name, object_key)}?{query_with_sig}"
|
||||
|
||||
|
||||
@@ -1026,6 +1043,7 @@ def bucket_handler(bucket_name: str) -> Response:
|
||||
try:
|
||||
storage.delete_bucket(bucket_name)
|
||||
_bucket_policies().delete_policy(bucket_name)
|
||||
_replication_manager().delete_rule(bucket_name)
|
||||
except StorageError as exc:
|
||||
code = "BucketNotEmpty" if "not empty" in str(exc) else "NoSuchBucket"
|
||||
status = 409 if code == "BucketNotEmpty" else 404
|
||||
@@ -1069,7 +1087,12 @@ def object_handler(bucket_name: str, object_key: str):
|
||||
_, error = _object_principal("write", bucket_name, object_key)
|
||||
if error:
|
||||
return error
|
||||
|
||||
stream = request.stream
|
||||
content_encoding = request.headers.get("Content-Encoding", "").lower()
|
||||
if "aws-chunked" in content_encoding:
|
||||
stream = AwsChunkedDecoder(stream)
|
||||
|
||||
metadata = _extract_request_metadata()
|
||||
try:
|
||||
meta = storage.put_object(
|
||||
@@ -1089,6 +1112,12 @@ def object_handler(bucket_name: str, object_key: str):
|
||||
)
|
||||
response = Response(status=200)
|
||||
response.headers["ETag"] = f'"{meta.etag}"'
|
||||
|
||||
# Trigger replication if not a replication request
|
||||
user_agent = request.headers.get("User-Agent", "")
|
||||
if "S3ReplicationAgent" not in user_agent:
|
||||
_replication_manager().trigger_replication(bucket_name, object_key, action="write")
|
||||
|
||||
return response
|
||||
|
||||
if request.method in {"GET", "HEAD"}:
|
||||
@@ -1123,6 +1152,12 @@ def object_handler(bucket_name: str, object_key: str):
|
||||
return error
|
||||
storage.delete_object(bucket_name, object_key)
|
||||
current_app.logger.info("Object deleted", extra={"bucket": bucket_name, "key": object_key})
|
||||
|
||||
# Trigger replication if not a replication request
|
||||
user_agent = request.headers.get("User-Agent", "")
|
||||
if "S3ReplicationAgent" not in user_agent:
|
||||
_replication_manager().trigger_replication(bucket_name, object_key, action="delete")
|
||||
|
||||
return Response(status=204)
|
||||
|
||||
|
||||
@@ -1243,3 +1278,79 @@ def head_object(bucket_name: str, object_key: str) -> Response:
|
||||
return _error_response("NoSuchKey", "Object not found", 404)
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
|
||||
|
||||
class AwsChunkedDecoder:
|
||||
"""Decodes aws-chunked encoded streams."""
|
||||
def __init__(self, stream):
|
||||
self.stream = stream
|
||||
self.buffer = b""
|
||||
self.chunk_remaining = 0
|
||||
self.finished = False
|
||||
|
||||
def read(self, size=-1):
|
||||
if self.finished:
|
||||
return b""
|
||||
|
||||
result = b""
|
||||
while size == -1 or len(result) < size:
|
||||
if self.chunk_remaining > 0:
|
||||
to_read = self.chunk_remaining
|
||||
if size != -1:
|
||||
to_read = min(to_read, size - len(result))
|
||||
|
||||
chunk = self.stream.read(to_read)
|
||||
if not chunk:
|
||||
raise IOError("Unexpected EOF in chunk data")
|
||||
|
||||
result += chunk
|
||||
self.chunk_remaining -= len(chunk)
|
||||
|
||||
if self.chunk_remaining == 0:
|
||||
# Read CRLF after chunk data
|
||||
crlf = self.stream.read(2)
|
||||
if crlf != b"\r\n":
|
||||
raise IOError("Malformed chunk: missing CRLF")
|
||||
else:
|
||||
# Read chunk size line
|
||||
line = b""
|
||||
while True:
|
||||
char = self.stream.read(1)
|
||||
if not char:
|
||||
if not line: # EOF at start of chunk size
|
||||
self.finished = True
|
||||
return result
|
||||
raise IOError("Unexpected EOF in chunk size")
|
||||
line += char
|
||||
if line.endswith(b"\r\n"):
|
||||
break
|
||||
|
||||
# Parse chunk size (hex)
|
||||
try:
|
||||
line_str = line.decode("ascii").strip()
|
||||
# Handle chunk-signature extension if present (e.g. "1000;chunk-signature=...")
|
||||
if ";" in line_str:
|
||||
line_str = line_str.split(";")[0]
|
||||
chunk_size = int(line_str, 16)
|
||||
except ValueError:
|
||||
raise IOError(f"Invalid chunk size: {line}")
|
||||
|
||||
if chunk_size == 0:
|
||||
self.finished = True
|
||||
# Read trailers if any (until empty line)
|
||||
while True:
|
||||
line = b""
|
||||
while True:
|
||||
char = self.stream.read(1)
|
||||
if not char:
|
||||
break
|
||||
line += char
|
||||
if line.endswith(b"\r\n"):
|
||||
break
|
||||
if line == b"\r\n" or not line:
|
||||
break
|
||||
return result
|
||||
|
||||
self.chunk_remaining = chunk_size
|
||||
|
||||
return result
|
||||
|
||||
97
app/ui.py
97
app/ui.py
@@ -6,7 +6,9 @@ import uuid
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import boto3
|
||||
import requests
|
||||
from botocore.exceptions import ClientError
|
||||
from flask import (
|
||||
Blueprint,
|
||||
Response,
|
||||
@@ -38,6 +40,10 @@ def _storage() -> ObjectStorage:
|
||||
return current_app.extensions["object_storage"]
|
||||
|
||||
|
||||
def _replication_manager() -> ReplicationManager:
|
||||
return current_app.extensions["replication"]
|
||||
|
||||
|
||||
def _iam():
|
||||
return current_app.extensions["iam"]
|
||||
|
||||
@@ -494,6 +500,7 @@ def delete_bucket(bucket_name: str):
|
||||
_authorize_ui(principal, bucket_name, "delete")
|
||||
_storage().delete_bucket(bucket_name)
|
||||
_bucket_policies().delete_policy(bucket_name)
|
||||
_replication_manager().delete_rule(bucket_name)
|
||||
flash(f"Bucket '{bucket_name}' removed", "success")
|
||||
except (StorageError, IamError) as exc:
|
||||
flash(_friendly_error_message(exc), "danger")
|
||||
@@ -512,6 +519,7 @@ def delete_object(bucket_name: str, object_key: str):
|
||||
flash(f"Permanently deleted '{object_key}' and all versions", "success")
|
||||
else:
|
||||
_storage().delete_object(bucket_name, object_key)
|
||||
_replication_manager().trigger_replication(bucket_name, object_key, action="delete")
|
||||
flash(f"Deleted '{object_key}'", "success")
|
||||
except (IamError, StorageError) as exc:
|
||||
flash(_friendly_error_message(exc), "danger")
|
||||
@@ -572,6 +580,7 @@ def bulk_delete_objects(bucket_name: str):
|
||||
storage.purge_object(bucket_name, key)
|
||||
else:
|
||||
storage.delete_object(bucket_name, key)
|
||||
_replication_manager().trigger_replication(bucket_name, key, action="delete")
|
||||
deleted.append(key)
|
||||
except StorageError as exc:
|
||||
errors.append({"key": key, "error": str(exc)})
|
||||
@@ -701,10 +710,21 @@ def object_presign(bucket_name: str, object_key: str):
|
||||
_authorize_ui(principal, bucket_name, action, object_key=object_key)
|
||||
except IamError as exc:
|
||||
return jsonify({"error": str(exc)}), 403
|
||||
api_base = current_app.config["API_BASE_URL"].rstrip("/")
|
||||
|
||||
api_base = current_app.config.get("API_BASE_URL")
|
||||
if not api_base:
|
||||
api_base = "http://127.0.0.1:5000"
|
||||
api_base = api_base.rstrip("/")
|
||||
|
||||
url = f"{api_base}/presign/{bucket_name}/{object_key}"
|
||||
|
||||
headers = _api_headers()
|
||||
# Forward the host so the API knows the public URL
|
||||
headers["X-Forwarded-Host"] = request.host
|
||||
headers["X-Forwarded-Proto"] = request.scheme
|
||||
|
||||
try:
|
||||
response = requests.post(url, headers=_api_headers(), json=payload, timeout=5)
|
||||
response = requests.post(url, headers=headers, json=payload, timeout=5)
|
||||
except requests.RequestException as exc:
|
||||
return jsonify({"error": f"API unavailable: {exc}"}), 502
|
||||
try:
|
||||
@@ -926,6 +946,12 @@ def rotate_iam_secret(access_key: str):
|
||||
return redirect(url_for("ui.iam_dashboard"))
|
||||
try:
|
||||
new_secret = _iam().rotate_secret(access_key)
|
||||
# If rotating own key, update session immediately so subsequent API calls (like presign) work
|
||||
if principal and principal.access_key == access_key:
|
||||
creds = session.get("credentials", {})
|
||||
creds["secret_key"] = new_secret
|
||||
session["credentials"] = creds
|
||||
session.modified = True
|
||||
except IamError as exc:
|
||||
if request.accept_mimetypes.accept_json and not request.accept_mimetypes.accept_html:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
@@ -1064,6 +1090,73 @@ def create_connection():
|
||||
return redirect(url_for("ui.connections_dashboard"))
|
||||
|
||||
|
||||
@ui_bp.post("/connections/test")
|
||||
def test_connection():
|
||||
principal = _current_principal()
|
||||
try:
|
||||
_iam().authorize(principal, None, "iam:list_users")
|
||||
except IamError:
|
||||
return jsonify({"status": "error", "message": "Access denied"}), 403
|
||||
|
||||
data = request.get_json(silent=True) or request.form
|
||||
endpoint = data.get("endpoint_url", "").strip()
|
||||
access_key = data.get("access_key", "").strip()
|
||||
secret_key = data.get("secret_key", "").strip()
|
||||
region = data.get("region", "us-east-1").strip()
|
||||
|
||||
if not all([endpoint, access_key, secret_key]):
|
||||
return jsonify({"status": "error", "message": "Missing credentials"}), 400
|
||||
|
||||
try:
|
||||
s3 = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
region_name=region,
|
||||
)
|
||||
# Try to list buckets to verify credentials and endpoint
|
||||
s3.list_buckets()
|
||||
return jsonify({"status": "ok", "message": "Connection successful"})
|
||||
except Exception as e:
|
||||
return jsonify({"status": "error", "message": str(e)}), 400
|
||||
|
||||
|
||||
@ui_bp.post("/connections/<connection_id>/update")
|
||||
def update_connection(connection_id: str):
|
||||
principal = _current_principal()
|
||||
try:
|
||||
_iam().authorize(principal, None, "iam:list_users")
|
||||
except IamError:
|
||||
flash("Access denied", "danger")
|
||||
return redirect(url_for("ui.buckets_overview"))
|
||||
|
||||
conn = _connections().get(connection_id)
|
||||
if not conn:
|
||||
flash("Connection not found", "danger")
|
||||
return redirect(url_for("ui.connections_dashboard"))
|
||||
|
||||
name = request.form.get("name", "").strip()
|
||||
endpoint = request.form.get("endpoint_url", "").strip()
|
||||
access_key = request.form.get("access_key", "").strip()
|
||||
secret_key = request.form.get("secret_key", "").strip()
|
||||
region = request.form.get("region", "us-east-1").strip()
|
||||
|
||||
if not all([name, endpoint, access_key, secret_key]):
|
||||
flash("All fields are required", "danger")
|
||||
return redirect(url_for("ui.connections_dashboard"))
|
||||
|
||||
conn.name = name
|
||||
conn.endpoint_url = endpoint
|
||||
conn.access_key = access_key
|
||||
conn.secret_key = secret_key
|
||||
conn.region = region
|
||||
|
||||
_connections().save()
|
||||
flash(f"Connection '{name}' updated", "success")
|
||||
return redirect(url_for("ui.connections_dashboard"))
|
||||
|
||||
|
||||
@ui_bp.post("/connections/<connection_id>/delete")
|
||||
def delete_connection(connection_id: str):
|
||||
principal = _current_principal()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Central location for the application version string."""
|
||||
from __future__ import annotations
|
||||
|
||||
APP_VERSION = "0.1.0"
|
||||
APP_VERSION = "0.1.1"
|
||||
|
||||
|
||||
def get_version() -> str:
|
||||
|
||||
5
docker-entrypoint.sh
Normal file
5
docker-entrypoint.sh
Normal file
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
# Run both services using the python runner in production mode
|
||||
exec python run.py --prod
|
||||
25
docs.md
25
docs.md
@@ -77,12 +77,20 @@ The repo now tracks a human-friendly release string inside `app/version.py` (see
|
||||
| `SECRET_KEY` | `dev-secret-key` | Flask session key for UI auth. |
|
||||
| `IAM_CONFIG` | `<repo>/data/.myfsio.sys/config/iam.json` | Stores users, secrets, and inline policies. |
|
||||
| `BUCKET_POLICY_PATH` | `<repo>/data/.myfsio.sys/config/bucket_policies.json` | Bucket policy store (auto hot-reload). |
|
||||
| `API_BASE_URL` | `http://127.0.0.1:5000` | Used by the UI to hit API endpoints (presign/policy). |
|
||||
| `API_BASE_URL` | `None` | Used by the UI to hit API endpoints (presign/policy). If unset, the UI will auto-detect the host or use `X-Forwarded-*` headers. |
|
||||
| `AWS_REGION` | `us-east-1` | Region embedded in SigV4 credential scope. |
|
||||
| `AWS_SERVICE` | `s3` | Service string for SigV4. |
|
||||
|
||||
Set env vars (or pass overrides to `create_app`) to point the servers at custom paths.
|
||||
|
||||
### Proxy Configuration
|
||||
|
||||
If running behind a reverse proxy (e.g., Nginx, Cloudflare, or a tunnel), ensure the proxy sets the standard forwarding headers:
|
||||
- `X-Forwarded-Host`
|
||||
- `X-Forwarded-Proto`
|
||||
|
||||
The application automatically trusts these headers to generate correct presigned URLs (e.g., `https://s3.example.com/...` instead of `http://127.0.0.1:5000/...`). Alternatively, you can explicitly set `API_BASE_URL` to your public endpoint.
|
||||
|
||||
## 4. Authentication & IAM
|
||||
|
||||
1. On first boot, `data/.myfsio.sys/config/iam.json` is seeded with `localadmin / localadmin` that has wildcard access.
|
||||
@@ -262,6 +270,21 @@ Now, configure the primary instance to replicate to the target.
|
||||
aws --endpoint-url http://target-server:5002 s3 ls s3://backup-bucket
|
||||
```
|
||||
|
||||
### Bidirectional Replication (Active-Active)
|
||||
|
||||
To set up two-way replication (Server A ↔ Server B):
|
||||
|
||||
1. Follow the steps above to replicate **A → B**.
|
||||
2. Repeat the process on Server B to replicate **B → A**:
|
||||
- Create a connection on Server B pointing to Server A.
|
||||
- Enable replication on the target bucket on Server B.
|
||||
|
||||
**Loop Prevention**: The system automatically detects replication traffic using a custom User-Agent (`S3ReplicationAgent`). This prevents infinite loops where an object replicated from A to B is immediately replicated back to A.
|
||||
|
||||
**Deletes**: Deleting an object on one server will propagate the deletion to the other server.
|
||||
|
||||
**Note**: Deleting a bucket will automatically remove its associated replication configuration.
|
||||
|
||||
## 7. Running Tests
|
||||
|
||||
```bash
|
||||
|
||||
@@ -5,3 +5,4 @@ Flask-WTF>=1.2.1
|
||||
pytest>=7.4
|
||||
requests>=2.31
|
||||
boto3>=1.34
|
||||
waitress>=2.1.2
|
||||
|
||||
33
run.py
33
run.py
@@ -18,20 +18,28 @@ def _is_debug_enabled() -> bool:
|
||||
return os.getenv("FLASK_DEBUG", "0").lower() in ("1", "true", "yes")
|
||||
|
||||
|
||||
def serve_api(port: int) -> None:
|
||||
def serve_api(port: int, prod: bool = False) -> None:
|
||||
app = create_api_app()
|
||||
debug = _is_debug_enabled()
|
||||
if debug:
|
||||
warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning)
|
||||
app.run(host=_server_host(), port=port, debug=debug)
|
||||
if prod:
|
||||
from waitress import serve
|
||||
serve(app, host=_server_host(), port=port, ident="MyFSIO")
|
||||
else:
|
||||
debug = _is_debug_enabled()
|
||||
if debug:
|
||||
warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning)
|
||||
app.run(host=_server_host(), port=port, debug=debug)
|
||||
|
||||
|
||||
def serve_ui(port: int) -> None:
|
||||
def serve_ui(port: int, prod: bool = False) -> None:
|
||||
app = create_ui_app()
|
||||
debug = _is_debug_enabled()
|
||||
if debug:
|
||||
warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning)
|
||||
app.run(host=_server_host(), port=port, debug=debug)
|
||||
if prod:
|
||||
from waitress import serve
|
||||
serve(app, host=_server_host(), port=port, ident="MyFSIO")
|
||||
else:
|
||||
debug = _is_debug_enabled()
|
||||
if debug:
|
||||
warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning)
|
||||
app.run(host=_server_host(), port=port, debug=debug)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
@@ -39,18 +47,19 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--mode", choices=["api", "ui", "both"], default="both")
|
||||
parser.add_argument("--api-port", type=int, default=5000)
|
||||
parser.add_argument("--ui-port", type=int, default=5100)
|
||||
parser.add_argument("--prod", action="store_true", help="Run in production mode using Waitress")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.mode in {"api", "both"}:
|
||||
print(f"Starting API server on port {args.api_port}...")
|
||||
api_proc = Process(target=serve_api, args=(args.api_port,), daemon=True)
|
||||
api_proc = Process(target=serve_api, args=(args.api_port, args.prod), daemon=True)
|
||||
api_proc.start()
|
||||
else:
|
||||
api_proc = None
|
||||
|
||||
if args.mode in {"ui", "both"}:
|
||||
print(f"Starting UI server on port {args.ui_port}...")
|
||||
serve_ui(args.ui_port)
|
||||
serve_ui(args.ui_port, args.prod)
|
||||
elif api_proc:
|
||||
try:
|
||||
api_proc.join()
|
||||
|
||||
@@ -408,16 +408,16 @@
|
||||
</dl>
|
||||
</div>
|
||||
|
||||
<form method="POST" action="{{ url_for('ui.update_bucket_replication', bucket_name=bucket_name) }}" onsubmit="return confirm('Are you sure you want to disable replication?');">
|
||||
<input type="hidden" name="action" value="delete">
|
||||
<button type="submit" class="btn btn-danger">Disable Replication</button>
|
||||
</form>
|
||||
<button type="button" class="btn btn-danger" data-bs-toggle="modal" data-bs-target="#disableReplicationModal">
|
||||
Disable Replication
|
||||
</button>
|
||||
|
||||
{% else %}
|
||||
<p class="text-muted">Replication allows you to automatically copy new objects from this bucket to a bucket in another S3-compatible service.</p>
|
||||
|
||||
{% if connections %}
|
||||
<form method="POST" action="{{ url_for('ui.update_bucket_replication', bucket_name=bucket_name) }}">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
|
||||
<input type="hidden" name="action" value="create">
|
||||
|
||||
<div class="mb-3">
|
||||
@@ -434,7 +434,7 @@
|
||||
<div class="mb-3">
|
||||
<label for="target_bucket" class="form-label">Target Bucket Name</label>
|
||||
<input type="text" class="form-control" id="target_bucket" name="target_bucket" required placeholder="e.g. my-backup-bucket">
|
||||
<div class="form-text">The bucket on the remote service must already exist.</div>
|
||||
<div class="form-text">If the target bucket does not exist, it will be created automatically.</div>
|
||||
</div>
|
||||
|
||||
<button type="submit" class="btn btn-primary">Enable Replication</button>
|
||||
@@ -708,6 +708,30 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Disable Replication Modal -->
|
||||
<div class="modal fade" id="disableReplicationModal" tabindex="-1" aria-hidden="true">
|
||||
<div class="modal-dialog modal-dialog-centered">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title">Disable Replication</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<p>Are you sure you want to disable replication for this bucket?</p>
|
||||
<p class="text-muted small">Existing objects in the target bucket will remain, but new uploads will no longer be copied.</p>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<form method="POST" action="{{ url_for('ui.update_bucket_replication', bucket_name=bucket_name) }}">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
|
||||
<input type="hidden" name="action" value="delete">
|
||||
<button type="submit" class="btn btn-danger">Disable Replication</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block extra_scripts %}
|
||||
|
||||
@@ -12,12 +12,13 @@
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-4">
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
<div class="card shadow-sm">
|
||||
<div class="card-header fw-semibold">
|
||||
Add New Connection
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<form method="POST" action="{{ url_for('ui.create_connection') }}">
|
||||
<form method="POST" action="{{ url_for('ui.create_connection') }}" id="createConnectionForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
<div class="mb-3">
|
||||
<label for="name" class="form-label">Name</label>
|
||||
<input type="text" class="form-control" id="name" name="name" required placeholder="e.g. Production Backup">
|
||||
@@ -36,43 +37,69 @@
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="secret_key" class="form-label">Secret Key</label>
|
||||
<input type="password" class="form-control" id="secret_key" name="secret_key" required>
|
||||
<div class="input-group">
|
||||
<input type="password" class="form-control" id="secret_key" name="secret_key" required>
|
||||
<button class="btn btn-outline-secondary" type="button" onclick="togglePassword('secret_key')">
|
||||
<i class="bi bi-eye"></i> Show
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<button type="submit" class="btn btn-primary">Add Connection</button>
|
||||
<div class="d-grid gap-2">
|
||||
<button type="button" class="btn btn-outline-secondary" id="testConnectionBtn">Test Connection</button>
|
||||
<button type="submit" class="btn btn-primary">Add Connection</button>
|
||||
</div>
|
||||
<div id="testResult" class="mt-2"></div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-md-8">
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
<div class="card shadow-sm">
|
||||
<div class="card-header fw-semibold">
|
||||
Existing Connections
|
||||
</div>
|
||||
<div class="card-body">
|
||||
{% if connections %}
|
||||
<div class="table-responsive">
|
||||
<table class="table table-hover">
|
||||
<table class="table table-hover align-middle">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Endpoint</th>
|
||||
<th>Region</th>
|
||||
<th>Access Key</th>
|
||||
<th>Actions</th>
|
||||
<th class="text-end">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for conn in connections %}
|
||||
<tr>
|
||||
<td>{{ conn.name }}</td>
|
||||
<td>{{ conn.endpoint_url }}</td>
|
||||
<td>{{ conn.region }}</td>
|
||||
<td><code>{{ conn.access_key }}</code></td>
|
||||
<td>
|
||||
<form method="POST" action="{{ url_for('ui.delete_connection', connection_id=conn.id) }}" onsubmit="return confirm('Are you sure?');" style="display: inline;">
|
||||
<button type="submit" class="btn btn-sm btn-danger">Delete</button>
|
||||
</form>
|
||||
<td class="fw-medium">{{ conn.name }}</td>
|
||||
<td class="small text-muted">{{ conn.endpoint_url }}</td>
|
||||
<td><span class="badge bg-light text-dark border">{{ conn.region }}</span></td>
|
||||
<td><code class="small">{{ conn.access_key }}</code></td>
|
||||
<td class="text-end">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-primary"
|
||||
data-bs-toggle="modal"
|
||||
data-bs-target="#editConnectionModal"
|
||||
data-id="{{ conn.id }}"
|
||||
data-name="{{ conn.name }}"
|
||||
data-endpoint="{{ conn.endpoint_url }}"
|
||||
data-region="{{ conn.region }}"
|
||||
data-access="{{ conn.access_key }}"
|
||||
data-secret="{{ conn.secret_key }}">
|
||||
Edit
|
||||
</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-danger"
|
||||
data-bs-toggle="modal"
|
||||
data-bs-target="#deleteConnectionModal"
|
||||
data-id="{{ conn.id }}"
|
||||
data-name="{{ conn.name }}">
|
||||
Delete
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
@@ -80,10 +107,164 @@
|
||||
</table>
|
||||
</div>
|
||||
{% else %}
|
||||
<p class="text-muted text-center my-4">No remote connections configured.</p>
|
||||
<div class="text-center py-5 text-muted">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="48" height="48" fill="currentColor" class="bi bi-hdd-network mb-3" viewBox="0 0 16 16">
|
||||
<path d="M4.5 5a.5.5 0 1 0 0-1 .5.5 0 0 0 0 1zM3 4.5a.5.5 0 1 1-1 0 .5.5 0 0 1 1 0z"/>
|
||||
<path d="M0 4a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v1a2 2 0 0 1-2 2H8.5v3a1.5 1.5 0 0 1 1.5 1.5v3.375a1.125 1.125 0 0 1-1.125 1.125h-1.75a1.125 1.125 0 0 1-1.125-1.125V11.5A1.5 1.5 0 0 1 7.5 10V7H2a2 2 0 0 1-2-2V4zm1 0v1a1 1 0 0 0 1 1h12a1 1 0 0 0 1-1V4a1 1 0 0 0-1-1H2a1 1 0 0 0-1 1zm6 5v1.5a.5.5 0 0 0 .5.5h1.75a.5.5 0 0 0 .5-.5V10a.5.5 0 0 0-.5-.5H7.5a.5.5 0 0 0-.5.5z"/>
|
||||
</svg>
|
||||
<p>No remote connections configured.</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Edit Connection Modal -->
|
||||
<div class="modal fade" id="editConnectionModal" tabindex="-1" aria-hidden="true">
|
||||
<div class="modal-dialog">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title">Edit Connection</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<form method="POST" id="editConnectionForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
<div class="modal-body">
|
||||
<div class="mb-3">
|
||||
<label for="edit_name" class="form-label">Name</label>
|
||||
<input type="text" class="form-control" id="edit_name" name="name" required>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="edit_endpoint_url" class="form-label">Endpoint URL</label>
|
||||
<input type="url" class="form-control" id="edit_endpoint_url" name="endpoint_url" required>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="edit_region" class="form-label">Region</label>
|
||||
<input type="text" class="form-control" id="edit_region" name="region" required>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="edit_access_key" class="form-label">Access Key</label>
|
||||
<input type="text" class="form-control" id="edit_access_key" name="access_key" required>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="edit_secret_key" class="form-label">Secret Key</label>
|
||||
<div class="input-group">
|
||||
<input type="password" class="form-control" id="edit_secret_key" name="secret_key" required>
|
||||
<button class="btn btn-outline-secondary" type="button" onclick="togglePassword('edit_secret_key')">
|
||||
Show
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div id="editTestResult" class="mt-2"></div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-outline-secondary" id="editTestConnectionBtn">Test Connection</button>
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="submit" class="btn btn-primary">Save Changes</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Delete Connection Modal -->
|
||||
<div class="modal fade" id="deleteConnectionModal" tabindex="-1" aria-hidden="true">
|
||||
<div class="modal-dialog modal-dialog-centered">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title">Delete Connection</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<p>Are you sure you want to delete the connection <strong id="deleteConnectionName"></strong>?</p>
|
||||
<p class="text-muted small">This will stop any replication rules using this connection.</p>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<form method="POST" id="deleteConnectionForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
<button type="submit" class="btn btn-danger">Delete</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
function togglePassword(id) {
|
||||
const input = document.getElementById(id);
|
||||
if (input.type === "password") {
|
||||
input.type = "text";
|
||||
} else {
|
||||
input.type = "password";
|
||||
}
|
||||
}
|
||||
|
||||
// Test Connection Logic
|
||||
async function testConnection(formId, resultId) {
|
||||
const form = document.getElementById(formId);
|
||||
const resultDiv = document.getElementById(resultId);
|
||||
const formData = new FormData(form);
|
||||
const data = Object.fromEntries(formData.entries());
|
||||
|
||||
resultDiv.innerHTML = '<div class="text-info"><span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span> Testing...</div>';
|
||||
|
||||
try {
|
||||
const response = await fetch("{{ url_for('ui.test_connection') }}", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"X-CSRFToken": "{{ csrf_token() }}"
|
||||
},
|
||||
body: JSON.stringify(data)
|
||||
});
|
||||
|
||||
const result = await response.json();
|
||||
if (response.ok) {
|
||||
resultDiv.innerHTML = `<div class="text-success"><i class="bi bi-check-circle"></i> ${result.message}</div>`;
|
||||
} else {
|
||||
resultDiv.innerHTML = `<div class="text-danger"><i class="bi bi-exclamation-circle"></i> ${result.message}</div>`;
|
||||
}
|
||||
} catch (error) {
|
||||
resultDiv.innerHTML = `<div class="text-danger"><i class="bi bi-exclamation-circle"></i> Connection failed</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
document.getElementById('testConnectionBtn').addEventListener('click', () => {
|
||||
testConnection('createConnectionForm', 'testResult');
|
||||
});
|
||||
|
||||
document.getElementById('editTestConnectionBtn').addEventListener('click', () => {
|
||||
testConnection('editConnectionForm', 'editTestResult');
|
||||
});
|
||||
|
||||
// Modal Event Listeners
|
||||
const editModal = document.getElementById('editConnectionModal');
|
||||
editModal.addEventListener('show.bs.modal', event => {
|
||||
const button = event.relatedTarget;
|
||||
const id = button.getAttribute('data-id');
|
||||
|
||||
document.getElementById('edit_name').value = button.getAttribute('data-name');
|
||||
document.getElementById('edit_endpoint_url').value = button.getAttribute('data-endpoint');
|
||||
document.getElementById('edit_region').value = button.getAttribute('data-region');
|
||||
document.getElementById('edit_access_key').value = button.getAttribute('data-access');
|
||||
document.getElementById('edit_secret_key').value = button.getAttribute('data-secret');
|
||||
document.getElementById('editTestResult').innerHTML = '';
|
||||
|
||||
const form = document.getElementById('editConnectionForm');
|
||||
form.action = "{{ url_for('ui.update_connection', connection_id='CONN_ID') }}".replace('CONN_ID', id);
|
||||
});
|
||||
|
||||
const deleteModal = document.getElementById('deleteConnectionModal');
|
||||
deleteModal.addEventListener('show.bs.modal', event => {
|
||||
const button = event.relatedTarget;
|
||||
const id = button.getAttribute('data-id');
|
||||
const name = button.getAttribute('data-name');
|
||||
|
||||
document.getElementById('deleteConnectionName').textContent = name;
|
||||
const form = document.getElementById('deleteConnectionForm');
|
||||
form.action = "{{ url_for('ui.delete_connection', connection_id='CONN_ID') }}".replace('CONN_ID', id);
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
@@ -290,6 +290,18 @@ s3.complete_multipart_upload(
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Bidirectional Replication (Active-Active)</h3>
|
||||
<p class="small text-muted">To set up two-way replication (Server A ↔ Server B):</p>
|
||||
<ol class="docs-steps mb-3">
|
||||
<li>Follow the steps above to replicate <strong>A → B</strong>.</li>
|
||||
<li>Repeat the process on Server B to replicate <strong>B → A</strong> (create a connection to A, enable rule).</li>
|
||||
</ol>
|
||||
<p class="small text-muted mb-0">
|
||||
<strong>Loop Prevention:</strong> The system automatically detects replication traffic using a custom User-Agent (<code>S3ReplicationAgent</code>). This prevents infinite loops where an object replicated from A to B is immediately replicated back to A.
|
||||
<br>
|
||||
<strong>Deletes:</strong> Deleting an object on one server will propagate the deletion to the other server.
|
||||
</p>
|
||||
</div>
|
||||
</article>
|
||||
<article id="troubleshooting" class="card shadow-sm docs-section">
|
||||
@@ -330,8 +342,8 @@ s3.complete_multipart_upload(
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Requests hit the wrong host</td>
|
||||
<td><code>API_BASE_URL</code> not updated after tunneling/forwarding</td>
|
||||
<td>Set <code>API_BASE_URL</code> in your shell or <code>.env</code> to match the published host.</td>
|
||||
<td>Proxy headers missing or <code>API_BASE_URL</code> incorrect</td>
|
||||
<td>Ensure your proxy sends <code>X-Forwarded-Host</code>/<code>Proto</code> headers, or explicitly set <code>API_BASE_URL</code> to your public domain.</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
@@ -286,9 +286,6 @@
|
||||
</div>
|
||||
<div class="modal-body" id="rotateSecretConfirm">
|
||||
<p>Are you sure you want to rotate the secret key for <strong id="rotateUserLabel"></strong>?</p>
|
||||
<div id="rotateSelfWarning" class="alert alert-warning d-none">
|
||||
<strong>Warning:</strong> You are rotating your own secret key. You will need to sign in again with the new key.
|
||||
</div>
|
||||
<div class="alert alert-warning mb-0">
|
||||
The old secret key will stop working immediately. Any applications using it must be updated.
|
||||
</div>
|
||||
@@ -474,7 +471,6 @@
|
||||
const rotateSecretResult = document.getElementById('rotateSecretResult');
|
||||
const newSecretKeyInput = document.getElementById('newSecretKey');
|
||||
const copyNewSecretBtn = document.getElementById('copyNewSecret');
|
||||
const rotateSelfWarning = document.getElementById('rotateSelfWarning');
|
||||
let currentRotateKey = null;
|
||||
|
||||
document.querySelectorAll('[data-rotate-user]').forEach(btn => {
|
||||
@@ -482,12 +478,6 @@
|
||||
currentRotateKey = btn.dataset.rotateUser;
|
||||
rotateUserLabel.textContent = currentRotateKey;
|
||||
|
||||
if (currentRotateKey === currentUserKey) {
|
||||
rotateSelfWarning.classList.remove('d-none');
|
||||
} else {
|
||||
rotateSelfWarning.classList.add('d-none');
|
||||
}
|
||||
|
||||
// Reset Modal State
|
||||
rotateSecretConfirm.classList.remove('d-none');
|
||||
rotateSecretResult.classList.add('d-none');
|
||||
|
||||
Reference in New Issue
Block a user