Compare commits
23 Commits
f400cedf02
...
v0.1.1
| Author | SHA1 | Date | |
|---|---|---|---|
| fd8fb21517 | |||
| a095616569 | |||
| c6cbe822e1 | |||
| dddab6dbbc | |||
| 015c9cb52d | |||
| c8b1c33118 | |||
| ebef3dfa57 | |||
| 1116353d0f | |||
| e4b92a32a1 | |||
| 57c40dcdcc | |||
| 7d1735a59f | |||
| 9064f9d60e | |||
| 36c08b0ac1 | |||
| ec5d52f208 | |||
| 96de6164d1 | |||
| 8c00d7bd4b | |||
| a32d9dbd77 | |||
| fe3eacd2be | |||
| 471cf5a305 | |||
| 840fd176d3 | |||
| 5350d04ba5 | |||
| f2daa8a8a3 | |||
| e287b59645 |
13
.dockerignore
Normal file
13
.dockerignore
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
.git
|
||||||
|
.gitignore
|
||||||
|
.venv
|
||||||
|
__pycache__
|
||||||
|
*.pyc
|
||||||
|
*.pyo
|
||||||
|
*.pyd
|
||||||
|
.pytest_cache
|
||||||
|
.coverage
|
||||||
|
htmlcov
|
||||||
|
logs
|
||||||
|
data
|
||||||
|
tmp
|
||||||
11
Dockerfile
11
Dockerfile
@@ -16,9 +16,14 @@ RUN pip install --no-cache-dir -r requirements.txt
|
|||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Drop privileges
|
# Make entrypoint executable
|
||||||
RUN useradd -m -u 1000 myfsio \
|
RUN chmod +x docker-entrypoint.sh
|
||||||
|
|
||||||
|
# Create data directory and set permissions
|
||||||
|
RUN mkdir -p /app/data \
|
||||||
|
&& useradd -m -u 1000 myfsio \
|
||||||
&& chown -R myfsio:myfsio /app
|
&& chown -R myfsio:myfsio /app
|
||||||
|
|
||||||
USER myfsio
|
USER myfsio
|
||||||
|
|
||||||
EXPOSE 5000 5100
|
EXPOSE 5000 5100
|
||||||
@@ -29,4 +34,4 @@ ENV APP_HOST=0.0.0.0 \
|
|||||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||||
CMD python -c "import requests; requests.get('http://localhost:5000/healthz', timeout=2)"
|
CMD python -c "import requests; requests.get('http://localhost:5000/healthz', timeout=2)"
|
||||||
|
|
||||||
CMD ["python", "run.py", "--mode", "both"]
|
CMD ["./docker-entrypoint.sh"]
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import sys
|
||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
from logging.handlers import RotatingFileHandler
|
from logging.handlers import RotatingFileHandler
|
||||||
@@ -12,6 +13,7 @@ from typing import Any, Dict, Optional
|
|||||||
from flask import Flask, g, has_request_context, redirect, render_template, request, url_for
|
from flask import Flask, g, has_request_context, redirect, render_template, request, url_for
|
||||||
from flask_cors import CORS
|
from flask_cors import CORS
|
||||||
from flask_wtf.csrf import CSRFError
|
from flask_wtf.csrf import CSRFError
|
||||||
|
from werkzeug.middleware.proxy_fix import ProxyFix
|
||||||
|
|
||||||
from .bucket_policies import BucketPolicyStore
|
from .bucket_policies import BucketPolicyStore
|
||||||
from .config import AppConfig
|
from .config import AppConfig
|
||||||
@@ -33,7 +35,11 @@ def create_app(
|
|||||||
"""Create and configure the Flask application."""
|
"""Create and configure the Flask application."""
|
||||||
config = AppConfig.from_env(test_config)
|
config = AppConfig.from_env(test_config)
|
||||||
|
|
||||||
project_root = Path(__file__).resolve().parent.parent
|
if getattr(sys, "frozen", False):
|
||||||
|
project_root = Path(sys._MEIPASS)
|
||||||
|
else:
|
||||||
|
project_root = Path(__file__).resolve().parent.parent
|
||||||
|
|
||||||
app = Flask(
|
app = Flask(
|
||||||
__name__,
|
__name__,
|
||||||
static_folder=str(project_root / "static"),
|
static_folder=str(project_root / "static"),
|
||||||
@@ -47,6 +53,9 @@ def create_app(
|
|||||||
if app.config.get("TESTING"):
|
if app.config.get("TESTING"):
|
||||||
app.config.setdefault("WTF_CSRF_ENABLED", False)
|
app.config.setdefault("WTF_CSRF_ENABLED", False)
|
||||||
|
|
||||||
|
# Trust X-Forwarded-* headers from proxies
|
||||||
|
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1)
|
||||||
|
|
||||||
_configure_cors(app)
|
_configure_cors(app)
|
||||||
_configure_logging(app)
|
_configure_logging(app)
|
||||||
|
|
||||||
@@ -167,23 +176,33 @@ class _RequestContextFilter(logging.Filter):
|
|||||||
|
|
||||||
|
|
||||||
def _configure_logging(app: Flask) -> None:
|
def _configure_logging(app: Flask) -> None:
|
||||||
log_file = Path(app.config["LOG_FILE"])
|
|
||||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
handler = RotatingFileHandler(
|
|
||||||
log_file,
|
|
||||||
maxBytes=int(app.config.get("LOG_MAX_BYTES", 5 * 1024 * 1024)),
|
|
||||||
backupCount=int(app.config.get("LOG_BACKUP_COUNT", 3)),
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
formatter = logging.Formatter(
|
formatter = logging.Formatter(
|
||||||
"%(asctime)s | %(levelname)s | %(request_id)s | %(method)s %(path)s | %(message)s"
|
"%(asctime)s | %(levelname)s | %(request_id)s | %(method)s %(path)s | %(message)s"
|
||||||
)
|
)
|
||||||
handler.setFormatter(formatter)
|
|
||||||
handler.addFilter(_RequestContextFilter())
|
# Stream Handler (stdout) - Primary for Docker
|
||||||
|
stream_handler = logging.StreamHandler(sys.stdout)
|
||||||
|
stream_handler.setFormatter(formatter)
|
||||||
|
stream_handler.addFilter(_RequestContextFilter())
|
||||||
|
|
||||||
logger = app.logger
|
logger = app.logger
|
||||||
logger.handlers.clear()
|
logger.handlers.clear()
|
||||||
logger.addHandler(handler)
|
logger.addHandler(stream_handler)
|
||||||
|
|
||||||
|
# File Handler (optional, if configured)
|
||||||
|
if app.config.get("LOG_TO_FILE"):
|
||||||
|
log_file = Path(app.config["LOG_FILE"])
|
||||||
|
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
file_handler = RotatingFileHandler(
|
||||||
|
log_file,
|
||||||
|
maxBytes=int(app.config.get("LOG_MAX_BYTES", 5 * 1024 * 1024)),
|
||||||
|
backupCount=int(app.config.get("LOG_BACKUP_COUNT", 3)),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
file_handler.setFormatter(formatter)
|
||||||
|
file_handler.addFilter(_RequestContextFilter())
|
||||||
|
logger.addHandler(file_handler)
|
||||||
|
|
||||||
logger.setLevel(getattr(logging, app.config.get("LOG_LEVEL", "INFO"), logging.INFO))
|
logger.setLevel(getattr(logging, app.config.get("LOG_LEVEL", "INFO"), logging.INFO))
|
||||||
|
|
||||||
@app.before_request
|
@app.before_request
|
||||||
@@ -211,5 +230,4 @@ def _configure_logging(app: Flask) -> None:
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
response.headers["X-Request-Duration-ms"] = f"{duration_ms:.2f}"
|
response.headers["X-Request-Duration-ms"] = f"{duration_ms:.2f}"
|
||||||
response.headers["Server"] = "MyFISO"
|
|
||||||
return response
|
return response
|
||||||
|
|||||||
@@ -4,12 +4,18 @@ from __future__ import annotations
|
|||||||
import os
|
import os
|
||||||
import secrets
|
import secrets
|
||||||
import shutil
|
import shutil
|
||||||
|
import sys
|
||||||
import warnings
|
import warnings
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, Optional
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
if getattr(sys, "frozen", False):
|
||||||
|
# Running in a PyInstaller bundle
|
||||||
|
PROJECT_ROOT = Path(sys._MEIPASS)
|
||||||
|
else:
|
||||||
|
# Running in a normal Python environment
|
||||||
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
|
||||||
|
|
||||||
def _prepare_config_file(active_path: Path, legacy_path: Optional[Path] = None) -> Path:
|
def _prepare_config_file(active_path: Path, legacy_path: Optional[Path] = None) -> Path:
|
||||||
@@ -39,7 +45,7 @@ class AppConfig:
|
|||||||
secret_key: str
|
secret_key: str
|
||||||
iam_config_path: Path
|
iam_config_path: Path
|
||||||
bucket_policy_path: Path
|
bucket_policy_path: Path
|
||||||
api_base_url: str
|
api_base_url: Optional[str]
|
||||||
aws_region: str
|
aws_region: str
|
||||||
aws_service: str
|
aws_service: str
|
||||||
ui_enforce_bucket_policies: bool
|
ui_enforce_bucket_policies: bool
|
||||||
@@ -78,11 +84,25 @@ class AppConfig:
|
|||||||
multipart_min_part_size = int(_get("MULTIPART_MIN_PART_SIZE", 5 * 1024 * 1024))
|
multipart_min_part_size = int(_get("MULTIPART_MIN_PART_SIZE", 5 * 1024 * 1024))
|
||||||
default_secret = "dev-secret-key"
|
default_secret = "dev-secret-key"
|
||||||
secret_key = str(_get("SECRET_KEY", default_secret))
|
secret_key = str(_get("SECRET_KEY", default_secret))
|
||||||
|
|
||||||
|
# If using default/missing secret, try to load/persist a generated one from disk
|
||||||
|
# This ensures consistency across Gunicorn workers
|
||||||
if not secret_key or secret_key == default_secret:
|
if not secret_key or secret_key == default_secret:
|
||||||
generated = secrets.token_urlsafe(32)
|
secret_file = storage_root / ".myfsio.sys" / "config" / ".secret"
|
||||||
if secret_key == default_secret:
|
if secret_file.exists():
|
||||||
warnings.warn("Using insecure default SECRET_KEY. A random value has been generated; set SECRET_KEY for production", RuntimeWarning)
|
secret_key = secret_file.read_text().strip()
|
||||||
secret_key = generated
|
else:
|
||||||
|
generated = secrets.token_urlsafe(32)
|
||||||
|
if secret_key == default_secret:
|
||||||
|
warnings.warn("Using insecure default SECRET_KEY. A random value has been generated and persisted; set SECRET_KEY for production", RuntimeWarning)
|
||||||
|
try:
|
||||||
|
secret_file.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
secret_file.write_text(generated)
|
||||||
|
secret_key = generated
|
||||||
|
except OSError:
|
||||||
|
# Fallback if we can't write to disk (e.g. read-only fs)
|
||||||
|
secret_key = generated
|
||||||
|
|
||||||
iam_env_override = "IAM_CONFIG" in overrides or "IAM_CONFIG" in os.environ
|
iam_env_override = "IAM_CONFIG" in overrides or "IAM_CONFIG" in os.environ
|
||||||
bucket_policy_override = "BUCKET_POLICY_PATH" in overrides or "BUCKET_POLICY_PATH" in os.environ
|
bucket_policy_override = "BUCKET_POLICY_PATH" in overrides or "BUCKET_POLICY_PATH" in os.environ
|
||||||
|
|
||||||
@@ -100,7 +120,10 @@ class AppConfig:
|
|||||||
bucket_policy_path,
|
bucket_policy_path,
|
||||||
legacy_path=None if bucket_policy_override else PROJECT_ROOT / "data" / "bucket_policies.json",
|
legacy_path=None if bucket_policy_override else PROJECT_ROOT / "data" / "bucket_policies.json",
|
||||||
)
|
)
|
||||||
api_base_url = str(_get("API_BASE_URL", "http://127.0.0.1:5000"))
|
api_base_url = _get("API_BASE_URL", None)
|
||||||
|
if api_base_url:
|
||||||
|
api_base_url = str(api_base_url)
|
||||||
|
|
||||||
aws_region = str(_get("AWS_REGION", "us-east-1"))
|
aws_region = str(_get("AWS_REGION", "us-east-1"))
|
||||||
aws_service = str(_get("AWS_SERVICE", "s3"))
|
aws_service = str(_get("AWS_SERVICE", "s3"))
|
||||||
enforce_ui_policies = str(_get("UI_ENFORCE_BUCKET_POLICIES", "0")).lower() in {"1", "true", "yes", "on"}
|
enforce_ui_policies = str(_get("UI_ENFORCE_BUCKET_POLICIES", "0")).lower() in {"1", "true", "yes", "on"}
|
||||||
|
|||||||
13
app/iam.py
13
app/iam.py
@@ -77,10 +77,20 @@ class IamService:
|
|||||||
self._users: Dict[str, Dict[str, Any]] = {}
|
self._users: Dict[str, Dict[str, Any]] = {}
|
||||||
self._raw_config: Dict[str, Any] = {}
|
self._raw_config: Dict[str, Any] = {}
|
||||||
self._failed_attempts: Dict[str, Deque[datetime]] = {}
|
self._failed_attempts: Dict[str, Deque[datetime]] = {}
|
||||||
|
self._last_load_time = 0.0
|
||||||
self._load()
|
self._load()
|
||||||
|
|
||||||
|
def _maybe_reload(self) -> None:
|
||||||
|
"""Reload configuration if the file has changed on disk."""
|
||||||
|
try:
|
||||||
|
if self.config_path.stat().st_mtime > self._last_load_time:
|
||||||
|
self._load()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
# ---------------------- authz helpers ----------------------
|
# ---------------------- authz helpers ----------------------
|
||||||
def authenticate(self, access_key: str, secret_key: str) -> Principal:
|
def authenticate(self, access_key: str, secret_key: str) -> Principal:
|
||||||
|
self._maybe_reload()
|
||||||
access_key = (access_key or "").strip()
|
access_key = (access_key or "").strip()
|
||||||
secret_key = (secret_key or "").strip()
|
secret_key = (secret_key or "").strip()
|
||||||
if not access_key or not secret_key:
|
if not access_key or not secret_key:
|
||||||
@@ -135,12 +145,14 @@ class IamService:
|
|||||||
return int(max(0, self.auth_lockout_window.total_seconds() - elapsed))
|
return int(max(0, self.auth_lockout_window.total_seconds() - elapsed))
|
||||||
|
|
||||||
def principal_for_key(self, access_key: str) -> Principal:
|
def principal_for_key(self, access_key: str) -> Principal:
|
||||||
|
self._maybe_reload()
|
||||||
record = self._users.get(access_key)
|
record = self._users.get(access_key)
|
||||||
if not record:
|
if not record:
|
||||||
raise IamError("Unknown access key")
|
raise IamError("Unknown access key")
|
||||||
return self._build_principal(access_key, record)
|
return self._build_principal(access_key, record)
|
||||||
|
|
||||||
def secret_for_key(self, access_key: str) -> str:
|
def secret_for_key(self, access_key: str) -> str:
|
||||||
|
self._maybe_reload()
|
||||||
record = self._users.get(access_key)
|
record = self._users.get(access_key)
|
||||||
if not record:
|
if not record:
|
||||||
raise IamError("Unknown access key")
|
raise IamError("Unknown access key")
|
||||||
@@ -245,6 +257,7 @@ class IamService:
|
|||||||
# ---------------------- config helpers ----------------------
|
# ---------------------- config helpers ----------------------
|
||||||
def _load(self) -> None:
|
def _load(self) -> None:
|
||||||
try:
|
try:
|
||||||
|
self._last_load_time = self.config_path.stat().st_mtime
|
||||||
content = self.config_path.read_text(encoding='utf-8')
|
content = self.config_path.read_text(encoding='utf-8')
|
||||||
raw = json.loads(content)
|
raw = json.loads(content)
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import mimetypes
|
||||||
import threading
|
import threading
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
@@ -9,13 +10,17 @@ from pathlib import Path
|
|||||||
from typing import Dict, Optional
|
from typing import Dict, Optional
|
||||||
|
|
||||||
import boto3
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
from botocore.exceptions import ClientError
|
from botocore.exceptions import ClientError
|
||||||
|
from boto3.exceptions import S3UploadFailedError
|
||||||
|
|
||||||
from .connections import ConnectionStore, RemoteConnection
|
from .connections import ConnectionStore, RemoteConnection
|
||||||
from .storage import ObjectStorage
|
from .storage import ObjectStorage, StorageError
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
REPLICATION_USER_AGENT = "S3ReplicationAgent/1.0"
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ReplicationRule:
|
class ReplicationRule:
|
||||||
@@ -66,7 +71,26 @@ class ReplicationManager:
|
|||||||
del self._rules[bucket_name]
|
del self._rules[bucket_name]
|
||||||
self.save_rules()
|
self.save_rules()
|
||||||
|
|
||||||
def trigger_replication(self, bucket_name: str, object_key: str) -> None:
|
def create_remote_bucket(self, connection_id: str, bucket_name: str) -> None:
|
||||||
|
"""Create a bucket on the remote connection."""
|
||||||
|
connection = self.connections.get(connection_id)
|
||||||
|
if not connection:
|
||||||
|
raise ValueError(f"Connection {connection_id} not found")
|
||||||
|
|
||||||
|
try:
|
||||||
|
s3 = boto3.client(
|
||||||
|
"s3",
|
||||||
|
endpoint_url=connection.endpoint_url,
|
||||||
|
aws_access_key_id=connection.access_key,
|
||||||
|
aws_secret_access_key=connection.secret_key,
|
||||||
|
region_name=connection.region,
|
||||||
|
)
|
||||||
|
s3.create_bucket(Bucket=bucket_name)
|
||||||
|
except ClientError as e:
|
||||||
|
logger.error(f"Failed to create remote bucket {bucket_name}: {e}")
|
||||||
|
raise
|
||||||
|
|
||||||
|
def trigger_replication(self, bucket_name: str, object_key: str, action: str = "write") -> None:
|
||||||
rule = self.get_rule(bucket_name)
|
rule = self.get_rule(bucket_name)
|
||||||
if not rule or not rule.enabled:
|
if not rule or not rule.enabled:
|
||||||
return
|
return
|
||||||
@@ -76,42 +100,93 @@ class ReplicationManager:
|
|||||||
logger.warning(f"Replication skipped for {bucket_name}/{object_key}: Connection {rule.target_connection_id} not found")
|
logger.warning(f"Replication skipped for {bucket_name}/{object_key}: Connection {rule.target_connection_id} not found")
|
||||||
return
|
return
|
||||||
|
|
||||||
self._executor.submit(self._replicate_task, bucket_name, object_key, rule, connection)
|
self._executor.submit(self._replicate_task, bucket_name, object_key, rule, connection, action)
|
||||||
|
|
||||||
def _replicate_task(self, bucket_name: str, object_key: str, rule: ReplicationRule, conn: RemoteConnection) -> None:
|
def _replicate_task(self, bucket_name: str, object_key: str, rule: ReplicationRule, conn: RemoteConnection, action: str) -> None:
|
||||||
try:
|
try:
|
||||||
# 1. Get local file path
|
|
||||||
# Note: We are accessing internal storage structure here.
|
|
||||||
# Ideally storage.py should expose a 'get_file_path' or we read the stream.
|
|
||||||
# For efficiency, we'll try to read the file directly if we can, or use storage.get_object
|
|
||||||
|
|
||||||
# Using boto3 to upload
|
# Using boto3 to upload
|
||||||
|
config = Config(user_agent_extra=REPLICATION_USER_AGENT)
|
||||||
s3 = boto3.client(
|
s3 = boto3.client(
|
||||||
"s3",
|
"s3",
|
||||||
endpoint_url=conn.endpoint_url,
|
endpoint_url=conn.endpoint_url,
|
||||||
aws_access_key_id=conn.access_key,
|
aws_access_key_id=conn.access_key,
|
||||||
aws_secret_access_key=conn.secret_key,
|
aws_secret_access_key=conn.secret_key,
|
||||||
region_name=conn.region,
|
region_name=conn.region,
|
||||||
|
config=config,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if action == "delete":
|
||||||
|
try:
|
||||||
|
s3.delete_object(Bucket=rule.target_bucket, Key=object_key)
|
||||||
|
logger.info(f"Replicated DELETE {bucket_name}/{object_key} to {conn.name} ({rule.target_bucket})")
|
||||||
|
except ClientError as e:
|
||||||
|
logger.error(f"Replication DELETE failed for {bucket_name}/{object_key}: {e}")
|
||||||
|
return
|
||||||
|
|
||||||
|
# 1. Get local file path
|
||||||
|
# Note: We are accessing internal storage structure here.
|
||||||
|
# Ideally storage.py should expose a 'get_file_path' or we read the stream.
|
||||||
|
# For efficiency, we'll try to read the file directly if we can, or use storage.get_object
|
||||||
|
|
||||||
# We need the file content.
|
# We need the file content.
|
||||||
# Since ObjectStorage is filesystem based, let's get the stream.
|
# Since ObjectStorage is filesystem based, let's get the stream.
|
||||||
# We need to be careful about closing it.
|
# We need to be careful about closing it.
|
||||||
meta = self.storage.get_object_meta(bucket_name, object_key)
|
try:
|
||||||
if not meta:
|
path = self.storage.get_object_path(bucket_name, object_key)
|
||||||
|
except StorageError:
|
||||||
return
|
return
|
||||||
|
|
||||||
with self.storage.open_object(bucket_name, object_key) as f:
|
metadata = self.storage.get_object_metadata(bucket_name, object_key)
|
||||||
extra_args = {}
|
|
||||||
if meta.metadata:
|
extra_args = {}
|
||||||
extra_args["Metadata"] = meta.metadata
|
if metadata:
|
||||||
|
extra_args["Metadata"] = metadata
|
||||||
s3.upload_fileobj(
|
|
||||||
f,
|
# Guess content type to prevent corruption/wrong handling
|
||||||
rule.target_bucket,
|
content_type, _ = mimetypes.guess_type(path)
|
||||||
object_key,
|
file_size = path.stat().st_size
|
||||||
ExtraArgs=extra_args
|
|
||||||
)
|
logger.info(f"Replicating {bucket_name}/{object_key}: Size={file_size}, ContentType={content_type}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
with path.open("rb") as f:
|
||||||
|
s3.put_object(
|
||||||
|
Bucket=rule.target_bucket,
|
||||||
|
Key=object_key,
|
||||||
|
Body=f,
|
||||||
|
ContentLength=file_size,
|
||||||
|
ContentType=content_type or "application/octet-stream",
|
||||||
|
Metadata=metadata or {}
|
||||||
|
)
|
||||||
|
except (ClientError, S3UploadFailedError) as e:
|
||||||
|
# Check if it's a NoSuchBucket error (either direct or wrapped)
|
||||||
|
is_no_bucket = False
|
||||||
|
if isinstance(e, ClientError):
|
||||||
|
if e.response['Error']['Code'] == 'NoSuchBucket':
|
||||||
|
is_no_bucket = True
|
||||||
|
elif isinstance(e, S3UploadFailedError):
|
||||||
|
if "NoSuchBucket" in str(e):
|
||||||
|
is_no_bucket = True
|
||||||
|
|
||||||
|
if is_no_bucket:
|
||||||
|
logger.info(f"Target bucket {rule.target_bucket} not found. Attempting to create it.")
|
||||||
|
try:
|
||||||
|
s3.create_bucket(Bucket=rule.target_bucket)
|
||||||
|
# Retry upload
|
||||||
|
with path.open("rb") as f:
|
||||||
|
s3.put_object(
|
||||||
|
Bucket=rule.target_bucket,
|
||||||
|
Key=object_key,
|
||||||
|
Body=f,
|
||||||
|
ContentLength=file_size,
|
||||||
|
ContentType=content_type or "application/octet-stream",
|
||||||
|
Metadata=metadata or {}
|
||||||
|
)
|
||||||
|
except Exception as create_err:
|
||||||
|
logger.error(f"Failed to create target bucket {rule.target_bucket}: {create_err}")
|
||||||
|
raise e # Raise original error
|
||||||
|
else:
|
||||||
|
raise e
|
||||||
|
|
||||||
logger.info(f"Replicated {bucket_name}/{object_key} to {conn.name} ({rule.target_bucket})")
|
logger.info(f"Replicated {bucket_name}/{object_key} to {conn.name} ({rule.target_bucket})")
|
||||||
|
|
||||||
|
|||||||
119
app/s3_api.py
119
app/s3_api.py
@@ -8,7 +8,7 @@ import re
|
|||||||
import uuid
|
import uuid
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Any, Dict
|
from typing import Any, Dict
|
||||||
from urllib.parse import quote, urlencode
|
from urllib.parse import quote, urlencode, urlparse
|
||||||
from xml.etree.ElementTree import Element, SubElement, tostring, fromstring, ParseError
|
from xml.etree.ElementTree import Element, SubElement, tostring, fromstring, ParseError
|
||||||
|
|
||||||
from flask import Blueprint, Response, current_app, jsonify, request
|
from flask import Blueprint, Response, current_app, jsonify, request
|
||||||
@@ -17,6 +17,7 @@ from werkzeug.http import http_date
|
|||||||
from .bucket_policies import BucketPolicyStore
|
from .bucket_policies import BucketPolicyStore
|
||||||
from .extensions import limiter
|
from .extensions import limiter
|
||||||
from .iam import IamError, Principal
|
from .iam import IamError, Principal
|
||||||
|
from .replication import ReplicationManager
|
||||||
from .storage import ObjectStorage, StorageError
|
from .storage import ObjectStorage, StorageError
|
||||||
|
|
||||||
s3_api_bp = Blueprint("s3_api", __name__)
|
s3_api_bp = Blueprint("s3_api", __name__)
|
||||||
@@ -31,6 +32,9 @@ def _iam():
|
|||||||
return current_app.extensions["iam"]
|
return current_app.extensions["iam"]
|
||||||
|
|
||||||
|
|
||||||
|
def _replication_manager() -> ReplicationManager:
|
||||||
|
return current_app.extensions["replication"]
|
||||||
|
|
||||||
|
|
||||||
def _bucket_policies() -> BucketPolicyStore:
|
def _bucket_policies() -> BucketPolicyStore:
|
||||||
store: BucketPolicyStore = current_app.extensions["bucket_policies"]
|
store: BucketPolicyStore = current_app.extensions["bucket_policies"]
|
||||||
@@ -405,7 +409,11 @@ def _canonical_headers_from_request(headers: list[str]) -> str:
|
|||||||
lines = []
|
lines = []
|
||||||
for header in headers:
|
for header in headers:
|
||||||
if header == "host":
|
if header == "host":
|
||||||
value = request.host
|
api_base = current_app.config.get("API_BASE_URL")
|
||||||
|
if api_base:
|
||||||
|
value = urlparse(api_base).netloc
|
||||||
|
else:
|
||||||
|
value = request.host
|
||||||
else:
|
else:
|
||||||
value = request.headers.get(header, "")
|
value = request.headers.get(header, "")
|
||||||
canonical_value = " ".join(value.strip().split()) if value else ""
|
canonical_value = " ".join(value.strip().split()) if value else ""
|
||||||
@@ -468,7 +476,17 @@ def _generate_presigned_url(
|
|||||||
"X-Amz-Content-Sha256": "UNSIGNED-PAYLOAD",
|
"X-Amz-Content-Sha256": "UNSIGNED-PAYLOAD",
|
||||||
}
|
}
|
||||||
canonical_query = _encode_query_params(query_params)
|
canonical_query = _encode_query_params(query_params)
|
||||||
host = request.host
|
|
||||||
|
# Determine host and scheme from config or request
|
||||||
|
api_base = current_app.config.get("API_BASE_URL")
|
||||||
|
if api_base:
|
||||||
|
parsed = urlparse(api_base)
|
||||||
|
host = parsed.netloc
|
||||||
|
scheme = parsed.scheme
|
||||||
|
else:
|
||||||
|
host = request.headers.get("X-Forwarded-Host", request.host)
|
||||||
|
scheme = request.headers.get("X-Forwarded-Proto", request.scheme or "http")
|
||||||
|
|
||||||
canonical_headers = f"host:{host}\n"
|
canonical_headers = f"host:{host}\n"
|
||||||
canonical_request = "\n".join(
|
canonical_request = "\n".join(
|
||||||
[
|
[
|
||||||
@@ -492,7 +510,6 @@ def _generate_presigned_url(
|
|||||||
signing_key = _derive_signing_key(secret_key, date_stamp, region, service)
|
signing_key = _derive_signing_key(secret_key, date_stamp, region, service)
|
||||||
signature = hmac.new(signing_key, string_to_sign.encode(), hashlib.sha256).hexdigest()
|
signature = hmac.new(signing_key, string_to_sign.encode(), hashlib.sha256).hexdigest()
|
||||||
query_with_sig = canonical_query + f"&X-Amz-Signature={signature}"
|
query_with_sig = canonical_query + f"&X-Amz-Signature={signature}"
|
||||||
scheme = request.scheme or "http"
|
|
||||||
return f"{scheme}://{host}{_canonical_uri(bucket_name, object_key)}?{query_with_sig}"
|
return f"{scheme}://{host}{_canonical_uri(bucket_name, object_key)}?{query_with_sig}"
|
||||||
|
|
||||||
|
|
||||||
@@ -1026,6 +1043,7 @@ def bucket_handler(bucket_name: str) -> Response:
|
|||||||
try:
|
try:
|
||||||
storage.delete_bucket(bucket_name)
|
storage.delete_bucket(bucket_name)
|
||||||
_bucket_policies().delete_policy(bucket_name)
|
_bucket_policies().delete_policy(bucket_name)
|
||||||
|
_replication_manager().delete_rule(bucket_name)
|
||||||
except StorageError as exc:
|
except StorageError as exc:
|
||||||
code = "BucketNotEmpty" if "not empty" in str(exc) else "NoSuchBucket"
|
code = "BucketNotEmpty" if "not empty" in str(exc) else "NoSuchBucket"
|
||||||
status = 409 if code == "BucketNotEmpty" else 404
|
status = 409 if code == "BucketNotEmpty" else 404
|
||||||
@@ -1069,7 +1087,12 @@ def object_handler(bucket_name: str, object_key: str):
|
|||||||
_, error = _object_principal("write", bucket_name, object_key)
|
_, error = _object_principal("write", bucket_name, object_key)
|
||||||
if error:
|
if error:
|
||||||
return error
|
return error
|
||||||
|
|
||||||
stream = request.stream
|
stream = request.stream
|
||||||
|
content_encoding = request.headers.get("Content-Encoding", "").lower()
|
||||||
|
if "aws-chunked" in content_encoding:
|
||||||
|
stream = AwsChunkedDecoder(stream)
|
||||||
|
|
||||||
metadata = _extract_request_metadata()
|
metadata = _extract_request_metadata()
|
||||||
try:
|
try:
|
||||||
meta = storage.put_object(
|
meta = storage.put_object(
|
||||||
@@ -1089,6 +1112,12 @@ def object_handler(bucket_name: str, object_key: str):
|
|||||||
)
|
)
|
||||||
response = Response(status=200)
|
response = Response(status=200)
|
||||||
response.headers["ETag"] = f'"{meta.etag}"'
|
response.headers["ETag"] = f'"{meta.etag}"'
|
||||||
|
|
||||||
|
# Trigger replication if not a replication request
|
||||||
|
user_agent = request.headers.get("User-Agent", "")
|
||||||
|
if "S3ReplicationAgent" not in user_agent:
|
||||||
|
_replication_manager().trigger_replication(bucket_name, object_key, action="write")
|
||||||
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
if request.method in {"GET", "HEAD"}:
|
if request.method in {"GET", "HEAD"}:
|
||||||
@@ -1123,6 +1152,12 @@ def object_handler(bucket_name: str, object_key: str):
|
|||||||
return error
|
return error
|
||||||
storage.delete_object(bucket_name, object_key)
|
storage.delete_object(bucket_name, object_key)
|
||||||
current_app.logger.info("Object deleted", extra={"bucket": bucket_name, "key": object_key})
|
current_app.logger.info("Object deleted", extra={"bucket": bucket_name, "key": object_key})
|
||||||
|
|
||||||
|
# Trigger replication if not a replication request
|
||||||
|
user_agent = request.headers.get("User-Agent", "")
|
||||||
|
if "S3ReplicationAgent" not in user_agent:
|
||||||
|
_replication_manager().trigger_replication(bucket_name, object_key, action="delete")
|
||||||
|
|
||||||
return Response(status=204)
|
return Response(status=204)
|
||||||
|
|
||||||
|
|
||||||
@@ -1243,3 +1278,79 @@ def head_object(bucket_name: str, object_key: str) -> Response:
|
|||||||
return _error_response("NoSuchKey", "Object not found", 404)
|
return _error_response("NoSuchKey", "Object not found", 404)
|
||||||
except IamError as exc:
|
except IamError as exc:
|
||||||
return _error_response("AccessDenied", str(exc), 403)
|
return _error_response("AccessDenied", str(exc), 403)
|
||||||
|
|
||||||
|
|
||||||
|
class AwsChunkedDecoder:
|
||||||
|
"""Decodes aws-chunked encoded streams."""
|
||||||
|
def __init__(self, stream):
|
||||||
|
self.stream = stream
|
||||||
|
self.buffer = b""
|
||||||
|
self.chunk_remaining = 0
|
||||||
|
self.finished = False
|
||||||
|
|
||||||
|
def read(self, size=-1):
|
||||||
|
if self.finished:
|
||||||
|
return b""
|
||||||
|
|
||||||
|
result = b""
|
||||||
|
while size == -1 or len(result) < size:
|
||||||
|
if self.chunk_remaining > 0:
|
||||||
|
to_read = self.chunk_remaining
|
||||||
|
if size != -1:
|
||||||
|
to_read = min(to_read, size - len(result))
|
||||||
|
|
||||||
|
chunk = self.stream.read(to_read)
|
||||||
|
if not chunk:
|
||||||
|
raise IOError("Unexpected EOF in chunk data")
|
||||||
|
|
||||||
|
result += chunk
|
||||||
|
self.chunk_remaining -= len(chunk)
|
||||||
|
|
||||||
|
if self.chunk_remaining == 0:
|
||||||
|
# Read CRLF after chunk data
|
||||||
|
crlf = self.stream.read(2)
|
||||||
|
if crlf != b"\r\n":
|
||||||
|
raise IOError("Malformed chunk: missing CRLF")
|
||||||
|
else:
|
||||||
|
# Read chunk size line
|
||||||
|
line = b""
|
||||||
|
while True:
|
||||||
|
char = self.stream.read(1)
|
||||||
|
if not char:
|
||||||
|
if not line: # EOF at start of chunk size
|
||||||
|
self.finished = True
|
||||||
|
return result
|
||||||
|
raise IOError("Unexpected EOF in chunk size")
|
||||||
|
line += char
|
||||||
|
if line.endswith(b"\r\n"):
|
||||||
|
break
|
||||||
|
|
||||||
|
# Parse chunk size (hex)
|
||||||
|
try:
|
||||||
|
line_str = line.decode("ascii").strip()
|
||||||
|
# Handle chunk-signature extension if present (e.g. "1000;chunk-signature=...")
|
||||||
|
if ";" in line_str:
|
||||||
|
line_str = line_str.split(";")[0]
|
||||||
|
chunk_size = int(line_str, 16)
|
||||||
|
except ValueError:
|
||||||
|
raise IOError(f"Invalid chunk size: {line}")
|
||||||
|
|
||||||
|
if chunk_size == 0:
|
||||||
|
self.finished = True
|
||||||
|
# Read trailers if any (until empty line)
|
||||||
|
while True:
|
||||||
|
line = b""
|
||||||
|
while True:
|
||||||
|
char = self.stream.read(1)
|
||||||
|
if not char:
|
||||||
|
break
|
||||||
|
line += char
|
||||||
|
if line.endswith(b"\r\n"):
|
||||||
|
break
|
||||||
|
if line == b"\r\n" or not line:
|
||||||
|
break
|
||||||
|
return result
|
||||||
|
|
||||||
|
self.chunk_remaining = chunk_size
|
||||||
|
|
||||||
|
return result
|
||||||
|
|||||||
97
app/ui.py
97
app/ui.py
@@ -6,7 +6,9 @@ import uuid
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
import boto3
|
||||||
import requests
|
import requests
|
||||||
|
from botocore.exceptions import ClientError
|
||||||
from flask import (
|
from flask import (
|
||||||
Blueprint,
|
Blueprint,
|
||||||
Response,
|
Response,
|
||||||
@@ -38,6 +40,10 @@ def _storage() -> ObjectStorage:
|
|||||||
return current_app.extensions["object_storage"]
|
return current_app.extensions["object_storage"]
|
||||||
|
|
||||||
|
|
||||||
|
def _replication_manager() -> ReplicationManager:
|
||||||
|
return current_app.extensions["replication"]
|
||||||
|
|
||||||
|
|
||||||
def _iam():
|
def _iam():
|
||||||
return current_app.extensions["iam"]
|
return current_app.extensions["iam"]
|
||||||
|
|
||||||
@@ -494,6 +500,7 @@ def delete_bucket(bucket_name: str):
|
|||||||
_authorize_ui(principal, bucket_name, "delete")
|
_authorize_ui(principal, bucket_name, "delete")
|
||||||
_storage().delete_bucket(bucket_name)
|
_storage().delete_bucket(bucket_name)
|
||||||
_bucket_policies().delete_policy(bucket_name)
|
_bucket_policies().delete_policy(bucket_name)
|
||||||
|
_replication_manager().delete_rule(bucket_name)
|
||||||
flash(f"Bucket '{bucket_name}' removed", "success")
|
flash(f"Bucket '{bucket_name}' removed", "success")
|
||||||
except (StorageError, IamError) as exc:
|
except (StorageError, IamError) as exc:
|
||||||
flash(_friendly_error_message(exc), "danger")
|
flash(_friendly_error_message(exc), "danger")
|
||||||
@@ -512,6 +519,7 @@ def delete_object(bucket_name: str, object_key: str):
|
|||||||
flash(f"Permanently deleted '{object_key}' and all versions", "success")
|
flash(f"Permanently deleted '{object_key}' and all versions", "success")
|
||||||
else:
|
else:
|
||||||
_storage().delete_object(bucket_name, object_key)
|
_storage().delete_object(bucket_name, object_key)
|
||||||
|
_replication_manager().trigger_replication(bucket_name, object_key, action="delete")
|
||||||
flash(f"Deleted '{object_key}'", "success")
|
flash(f"Deleted '{object_key}'", "success")
|
||||||
except (IamError, StorageError) as exc:
|
except (IamError, StorageError) as exc:
|
||||||
flash(_friendly_error_message(exc), "danger")
|
flash(_friendly_error_message(exc), "danger")
|
||||||
@@ -572,6 +580,7 @@ def bulk_delete_objects(bucket_name: str):
|
|||||||
storage.purge_object(bucket_name, key)
|
storage.purge_object(bucket_name, key)
|
||||||
else:
|
else:
|
||||||
storage.delete_object(bucket_name, key)
|
storage.delete_object(bucket_name, key)
|
||||||
|
_replication_manager().trigger_replication(bucket_name, key, action="delete")
|
||||||
deleted.append(key)
|
deleted.append(key)
|
||||||
except StorageError as exc:
|
except StorageError as exc:
|
||||||
errors.append({"key": key, "error": str(exc)})
|
errors.append({"key": key, "error": str(exc)})
|
||||||
@@ -701,10 +710,21 @@ def object_presign(bucket_name: str, object_key: str):
|
|||||||
_authorize_ui(principal, bucket_name, action, object_key=object_key)
|
_authorize_ui(principal, bucket_name, action, object_key=object_key)
|
||||||
except IamError as exc:
|
except IamError as exc:
|
||||||
return jsonify({"error": str(exc)}), 403
|
return jsonify({"error": str(exc)}), 403
|
||||||
api_base = current_app.config["API_BASE_URL"].rstrip("/")
|
|
||||||
|
api_base = current_app.config.get("API_BASE_URL")
|
||||||
|
if not api_base:
|
||||||
|
api_base = "http://127.0.0.1:5000"
|
||||||
|
api_base = api_base.rstrip("/")
|
||||||
|
|
||||||
url = f"{api_base}/presign/{bucket_name}/{object_key}"
|
url = f"{api_base}/presign/{bucket_name}/{object_key}"
|
||||||
|
|
||||||
|
headers = _api_headers()
|
||||||
|
# Forward the host so the API knows the public URL
|
||||||
|
headers["X-Forwarded-Host"] = request.host
|
||||||
|
headers["X-Forwarded-Proto"] = request.scheme
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = requests.post(url, headers=_api_headers(), json=payload, timeout=5)
|
response = requests.post(url, headers=headers, json=payload, timeout=5)
|
||||||
except requests.RequestException as exc:
|
except requests.RequestException as exc:
|
||||||
return jsonify({"error": f"API unavailable: {exc}"}), 502
|
return jsonify({"error": f"API unavailable: {exc}"}), 502
|
||||||
try:
|
try:
|
||||||
@@ -926,6 +946,12 @@ def rotate_iam_secret(access_key: str):
|
|||||||
return redirect(url_for("ui.iam_dashboard"))
|
return redirect(url_for("ui.iam_dashboard"))
|
||||||
try:
|
try:
|
||||||
new_secret = _iam().rotate_secret(access_key)
|
new_secret = _iam().rotate_secret(access_key)
|
||||||
|
# If rotating own key, update session immediately so subsequent API calls (like presign) work
|
||||||
|
if principal and principal.access_key == access_key:
|
||||||
|
creds = session.get("credentials", {})
|
||||||
|
creds["secret_key"] = new_secret
|
||||||
|
session["credentials"] = creds
|
||||||
|
session.modified = True
|
||||||
except IamError as exc:
|
except IamError as exc:
|
||||||
if request.accept_mimetypes.accept_json and not request.accept_mimetypes.accept_html:
|
if request.accept_mimetypes.accept_json and not request.accept_mimetypes.accept_html:
|
||||||
return jsonify({"error": str(exc)}), 400
|
return jsonify({"error": str(exc)}), 400
|
||||||
@@ -1064,6 +1090,73 @@ def create_connection():
|
|||||||
return redirect(url_for("ui.connections_dashboard"))
|
return redirect(url_for("ui.connections_dashboard"))
|
||||||
|
|
||||||
|
|
||||||
|
@ui_bp.post("/connections/test")
|
||||||
|
def test_connection():
|
||||||
|
principal = _current_principal()
|
||||||
|
try:
|
||||||
|
_iam().authorize(principal, None, "iam:list_users")
|
||||||
|
except IamError:
|
||||||
|
return jsonify({"status": "error", "message": "Access denied"}), 403
|
||||||
|
|
||||||
|
data = request.get_json(silent=True) or request.form
|
||||||
|
endpoint = data.get("endpoint_url", "").strip()
|
||||||
|
access_key = data.get("access_key", "").strip()
|
||||||
|
secret_key = data.get("secret_key", "").strip()
|
||||||
|
region = data.get("region", "us-east-1").strip()
|
||||||
|
|
||||||
|
if not all([endpoint, access_key, secret_key]):
|
||||||
|
return jsonify({"status": "error", "message": "Missing credentials"}), 400
|
||||||
|
|
||||||
|
try:
|
||||||
|
s3 = boto3.client(
|
||||||
|
"s3",
|
||||||
|
endpoint_url=endpoint,
|
||||||
|
aws_access_key_id=access_key,
|
||||||
|
aws_secret_access_key=secret_key,
|
||||||
|
region_name=region,
|
||||||
|
)
|
||||||
|
# Try to list buckets to verify credentials and endpoint
|
||||||
|
s3.list_buckets()
|
||||||
|
return jsonify({"status": "ok", "message": "Connection successful"})
|
||||||
|
except Exception as e:
|
||||||
|
return jsonify({"status": "error", "message": str(e)}), 400
|
||||||
|
|
||||||
|
|
||||||
|
@ui_bp.post("/connections/<connection_id>/update")
|
||||||
|
def update_connection(connection_id: str):
|
||||||
|
principal = _current_principal()
|
||||||
|
try:
|
||||||
|
_iam().authorize(principal, None, "iam:list_users")
|
||||||
|
except IamError:
|
||||||
|
flash("Access denied", "danger")
|
||||||
|
return redirect(url_for("ui.buckets_overview"))
|
||||||
|
|
||||||
|
conn = _connections().get(connection_id)
|
||||||
|
if not conn:
|
||||||
|
flash("Connection not found", "danger")
|
||||||
|
return redirect(url_for("ui.connections_dashboard"))
|
||||||
|
|
||||||
|
name = request.form.get("name", "").strip()
|
||||||
|
endpoint = request.form.get("endpoint_url", "").strip()
|
||||||
|
access_key = request.form.get("access_key", "").strip()
|
||||||
|
secret_key = request.form.get("secret_key", "").strip()
|
||||||
|
region = request.form.get("region", "us-east-1").strip()
|
||||||
|
|
||||||
|
if not all([name, endpoint, access_key, secret_key]):
|
||||||
|
flash("All fields are required", "danger")
|
||||||
|
return redirect(url_for("ui.connections_dashboard"))
|
||||||
|
|
||||||
|
conn.name = name
|
||||||
|
conn.endpoint_url = endpoint
|
||||||
|
conn.access_key = access_key
|
||||||
|
conn.secret_key = secret_key
|
||||||
|
conn.region = region
|
||||||
|
|
||||||
|
_connections().save()
|
||||||
|
flash(f"Connection '{name}' updated", "success")
|
||||||
|
return redirect(url_for("ui.connections_dashboard"))
|
||||||
|
|
||||||
|
|
||||||
@ui_bp.post("/connections/<connection_id>/delete")
|
@ui_bp.post("/connections/<connection_id>/delete")
|
||||||
def delete_connection(connection_id: str):
|
def delete_connection(connection_id: str):
|
||||||
principal = _current_principal()
|
principal = _current_principal()
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
"""Central location for the application version string."""
|
"""Central location for the application version string."""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
APP_VERSION = "0.1.0"
|
APP_VERSION = "0.1.1"
|
||||||
|
|
||||||
|
|
||||||
def get_version() -> str:
|
def get_version() -> str:
|
||||||
|
|||||||
5
docker-entrypoint.sh
Normal file
5
docker-entrypoint.sh
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Run both services using the python runner in production mode
|
||||||
|
exec python run.py --prod
|
||||||
25
docs.md
25
docs.md
@@ -77,12 +77,20 @@ The repo now tracks a human-friendly release string inside `app/version.py` (see
|
|||||||
| `SECRET_KEY` | `dev-secret-key` | Flask session key for UI auth. |
|
| `SECRET_KEY` | `dev-secret-key` | Flask session key for UI auth. |
|
||||||
| `IAM_CONFIG` | `<repo>/data/.myfsio.sys/config/iam.json` | Stores users, secrets, and inline policies. |
|
| `IAM_CONFIG` | `<repo>/data/.myfsio.sys/config/iam.json` | Stores users, secrets, and inline policies. |
|
||||||
| `BUCKET_POLICY_PATH` | `<repo>/data/.myfsio.sys/config/bucket_policies.json` | Bucket policy store (auto hot-reload). |
|
| `BUCKET_POLICY_PATH` | `<repo>/data/.myfsio.sys/config/bucket_policies.json` | Bucket policy store (auto hot-reload). |
|
||||||
| `API_BASE_URL` | `http://127.0.0.1:5000` | Used by the UI to hit API endpoints (presign/policy). |
|
| `API_BASE_URL` | `None` | Used by the UI to hit API endpoints (presign/policy). If unset, the UI will auto-detect the host or use `X-Forwarded-*` headers. |
|
||||||
| `AWS_REGION` | `us-east-1` | Region embedded in SigV4 credential scope. |
|
| `AWS_REGION` | `us-east-1` | Region embedded in SigV4 credential scope. |
|
||||||
| `AWS_SERVICE` | `s3` | Service string for SigV4. |
|
| `AWS_SERVICE` | `s3` | Service string for SigV4. |
|
||||||
|
|
||||||
Set env vars (or pass overrides to `create_app`) to point the servers at custom paths.
|
Set env vars (or pass overrides to `create_app`) to point the servers at custom paths.
|
||||||
|
|
||||||
|
### Proxy Configuration
|
||||||
|
|
||||||
|
If running behind a reverse proxy (e.g., Nginx, Cloudflare, or a tunnel), ensure the proxy sets the standard forwarding headers:
|
||||||
|
- `X-Forwarded-Host`
|
||||||
|
- `X-Forwarded-Proto`
|
||||||
|
|
||||||
|
The application automatically trusts these headers to generate correct presigned URLs (e.g., `https://s3.example.com/...` instead of `http://127.0.0.1:5000/...`). Alternatively, you can explicitly set `API_BASE_URL` to your public endpoint.
|
||||||
|
|
||||||
## 4. Authentication & IAM
|
## 4. Authentication & IAM
|
||||||
|
|
||||||
1. On first boot, `data/.myfsio.sys/config/iam.json` is seeded with `localadmin / localadmin` that has wildcard access.
|
1. On first boot, `data/.myfsio.sys/config/iam.json` is seeded with `localadmin / localadmin` that has wildcard access.
|
||||||
@@ -262,6 +270,21 @@ Now, configure the primary instance to replicate to the target.
|
|||||||
aws --endpoint-url http://target-server:5002 s3 ls s3://backup-bucket
|
aws --endpoint-url http://target-server:5002 s3 ls s3://backup-bucket
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Bidirectional Replication (Active-Active)
|
||||||
|
|
||||||
|
To set up two-way replication (Server A ↔ Server B):
|
||||||
|
|
||||||
|
1. Follow the steps above to replicate **A → B**.
|
||||||
|
2. Repeat the process on Server B to replicate **B → A**:
|
||||||
|
- Create a connection on Server B pointing to Server A.
|
||||||
|
- Enable replication on the target bucket on Server B.
|
||||||
|
|
||||||
|
**Loop Prevention**: The system automatically detects replication traffic using a custom User-Agent (`S3ReplicationAgent`). This prevents infinite loops where an object replicated from A to B is immediately replicated back to A.
|
||||||
|
|
||||||
|
**Deletes**: Deleting an object on one server will propagate the deletion to the other server.
|
||||||
|
|
||||||
|
**Note**: Deleting a bucket will automatically remove its associated replication configuration.
|
||||||
|
|
||||||
## 7. Running Tests
|
## 7. Running Tests
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -5,3 +5,4 @@ Flask-WTF>=1.2.1
|
|||||||
pytest>=7.4
|
pytest>=7.4
|
||||||
requests>=2.31
|
requests>=2.31
|
||||||
boto3>=1.34
|
boto3>=1.34
|
||||||
|
waitress>=2.1.2
|
||||||
|
|||||||
33
run.py
33
run.py
@@ -18,20 +18,28 @@ def _is_debug_enabled() -> bool:
|
|||||||
return os.getenv("FLASK_DEBUG", "0").lower() in ("1", "true", "yes")
|
return os.getenv("FLASK_DEBUG", "0").lower() in ("1", "true", "yes")
|
||||||
|
|
||||||
|
|
||||||
def serve_api(port: int) -> None:
|
def serve_api(port: int, prod: bool = False) -> None:
|
||||||
app = create_api_app()
|
app = create_api_app()
|
||||||
debug = _is_debug_enabled()
|
if prod:
|
||||||
if debug:
|
from waitress import serve
|
||||||
warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning)
|
serve(app, host=_server_host(), port=port, ident="MyFSIO")
|
||||||
app.run(host=_server_host(), port=port, debug=debug)
|
else:
|
||||||
|
debug = _is_debug_enabled()
|
||||||
|
if debug:
|
||||||
|
warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning)
|
||||||
|
app.run(host=_server_host(), port=port, debug=debug)
|
||||||
|
|
||||||
|
|
||||||
def serve_ui(port: int) -> None:
|
def serve_ui(port: int, prod: bool = False) -> None:
|
||||||
app = create_ui_app()
|
app = create_ui_app()
|
||||||
debug = _is_debug_enabled()
|
if prod:
|
||||||
if debug:
|
from waitress import serve
|
||||||
warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning)
|
serve(app, host=_server_host(), port=port, ident="MyFSIO")
|
||||||
app.run(host=_server_host(), port=port, debug=debug)
|
else:
|
||||||
|
debug = _is_debug_enabled()
|
||||||
|
if debug:
|
||||||
|
warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning)
|
||||||
|
app.run(host=_server_host(), port=port, debug=debug)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
@@ -39,18 +47,19 @@ if __name__ == "__main__":
|
|||||||
parser.add_argument("--mode", choices=["api", "ui", "both"], default="both")
|
parser.add_argument("--mode", choices=["api", "ui", "both"], default="both")
|
||||||
parser.add_argument("--api-port", type=int, default=5000)
|
parser.add_argument("--api-port", type=int, default=5000)
|
||||||
parser.add_argument("--ui-port", type=int, default=5100)
|
parser.add_argument("--ui-port", type=int, default=5100)
|
||||||
|
parser.add_argument("--prod", action="store_true", help="Run in production mode using Waitress")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
if args.mode in {"api", "both"}:
|
if args.mode in {"api", "both"}:
|
||||||
print(f"Starting API server on port {args.api_port}...")
|
print(f"Starting API server on port {args.api_port}...")
|
||||||
api_proc = Process(target=serve_api, args=(args.api_port,), daemon=True)
|
api_proc = Process(target=serve_api, args=(args.api_port, args.prod), daemon=True)
|
||||||
api_proc.start()
|
api_proc.start()
|
||||||
else:
|
else:
|
||||||
api_proc = None
|
api_proc = None
|
||||||
|
|
||||||
if args.mode in {"ui", "both"}:
|
if args.mode in {"ui", "both"}:
|
||||||
print(f"Starting UI server on port {args.ui_port}...")
|
print(f"Starting UI server on port {args.ui_port}...")
|
||||||
serve_ui(args.ui_port)
|
serve_ui(args.ui_port, args.prod)
|
||||||
elif api_proc:
|
elif api_proc:
|
||||||
try:
|
try:
|
||||||
api_proc.join()
|
api_proc.join()
|
||||||
|
|||||||
@@ -408,16 +408,16 @@
|
|||||||
</dl>
|
</dl>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<form method="POST" action="{{ url_for('ui.update_bucket_replication', bucket_name=bucket_name) }}" onsubmit="return confirm('Are you sure you want to disable replication?');">
|
<button type="button" class="btn btn-danger" data-bs-toggle="modal" data-bs-target="#disableReplicationModal">
|
||||||
<input type="hidden" name="action" value="delete">
|
Disable Replication
|
||||||
<button type="submit" class="btn btn-danger">Disable Replication</button>
|
</button>
|
||||||
</form>
|
|
||||||
|
|
||||||
{% else %}
|
{% else %}
|
||||||
<p class="text-muted">Replication allows you to automatically copy new objects from this bucket to a bucket in another S3-compatible service.</p>
|
<p class="text-muted">Replication allows you to automatically copy new objects from this bucket to a bucket in another S3-compatible service.</p>
|
||||||
|
|
||||||
{% if connections %}
|
{% if connections %}
|
||||||
<form method="POST" action="{{ url_for('ui.update_bucket_replication', bucket_name=bucket_name) }}">
|
<form method="POST" action="{{ url_for('ui.update_bucket_replication', bucket_name=bucket_name) }}">
|
||||||
|
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
|
||||||
<input type="hidden" name="action" value="create">
|
<input type="hidden" name="action" value="create">
|
||||||
|
|
||||||
<div class="mb-3">
|
<div class="mb-3">
|
||||||
@@ -434,7 +434,7 @@
|
|||||||
<div class="mb-3">
|
<div class="mb-3">
|
||||||
<label for="target_bucket" class="form-label">Target Bucket Name</label>
|
<label for="target_bucket" class="form-label">Target Bucket Name</label>
|
||||||
<input type="text" class="form-control" id="target_bucket" name="target_bucket" required placeholder="e.g. my-backup-bucket">
|
<input type="text" class="form-control" id="target_bucket" name="target_bucket" required placeholder="e.g. my-backup-bucket">
|
||||||
<div class="form-text">The bucket on the remote service must already exist.</div>
|
<div class="form-text">If the target bucket does not exist, it will be created automatically.</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<button type="submit" class="btn btn-primary">Enable Replication</button>
|
<button type="submit" class="btn btn-primary">Enable Replication</button>
|
||||||
@@ -708,6 +708,30 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Disable Replication Modal -->
|
||||||
|
<div class="modal fade" id="disableReplicationModal" tabindex="-1" aria-hidden="true">
|
||||||
|
<div class="modal-dialog modal-dialog-centered">
|
||||||
|
<div class="modal-content">
|
||||||
|
<div class="modal-header">
|
||||||
|
<h5 class="modal-title">Disable Replication</h5>
|
||||||
|
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||||
|
</div>
|
||||||
|
<div class="modal-body">
|
||||||
|
<p>Are you sure you want to disable replication for this bucket?</p>
|
||||||
|
<p class="text-muted small">Existing objects in the target bucket will remain, but new uploads will no longer be copied.</p>
|
||||||
|
</div>
|
||||||
|
<div class="modal-footer">
|
||||||
|
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||||
|
<form method="POST" action="{{ url_for('ui.update_bucket_replication', bucket_name=bucket_name) }}">
|
||||||
|
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
|
||||||
|
<input type="hidden" name="action" value="delete">
|
||||||
|
<button type="submit" class="btn btn-danger">Disable Replication</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|
||||||
{% block extra_scripts %}
|
{% block extra_scripts %}
|
||||||
|
|||||||
@@ -12,12 +12,13 @@
|
|||||||
|
|
||||||
<div class="row">
|
<div class="row">
|
||||||
<div class="col-md-4">
|
<div class="col-md-4">
|
||||||
<div class="card">
|
<div class="card shadow-sm">
|
||||||
<div class="card-header">
|
<div class="card-header fw-semibold">
|
||||||
Add New Connection
|
Add New Connection
|
||||||
</div>
|
</div>
|
||||||
<div class="card-body">
|
<div class="card-body">
|
||||||
<form method="POST" action="{{ url_for('ui.create_connection') }}">
|
<form method="POST" action="{{ url_for('ui.create_connection') }}" id="createConnectionForm">
|
||||||
|
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||||
<div class="mb-3">
|
<div class="mb-3">
|
||||||
<label for="name" class="form-label">Name</label>
|
<label for="name" class="form-label">Name</label>
|
||||||
<input type="text" class="form-control" id="name" name="name" required placeholder="e.g. Production Backup">
|
<input type="text" class="form-control" id="name" name="name" required placeholder="e.g. Production Backup">
|
||||||
@@ -36,43 +37,69 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="mb-3">
|
<div class="mb-3">
|
||||||
<label for="secret_key" class="form-label">Secret Key</label>
|
<label for="secret_key" class="form-label">Secret Key</label>
|
||||||
<input type="password" class="form-control" id="secret_key" name="secret_key" required>
|
<div class="input-group">
|
||||||
|
<input type="password" class="form-control" id="secret_key" name="secret_key" required>
|
||||||
|
<button class="btn btn-outline-secondary" type="button" onclick="togglePassword('secret_key')">
|
||||||
|
<i class="bi bi-eye"></i> Show
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<button type="submit" class="btn btn-primary">Add Connection</button>
|
<div class="d-grid gap-2">
|
||||||
|
<button type="button" class="btn btn-outline-secondary" id="testConnectionBtn">Test Connection</button>
|
||||||
|
<button type="submit" class="btn btn-primary">Add Connection</button>
|
||||||
|
</div>
|
||||||
|
<div id="testResult" class="mt-2"></div>
|
||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="col-md-8">
|
<div class="col-md-8">
|
||||||
<div class="card">
|
<div class="card shadow-sm">
|
||||||
<div class="card-header">
|
<div class="card-header fw-semibold">
|
||||||
Existing Connections
|
Existing Connections
|
||||||
</div>
|
</div>
|
||||||
<div class="card-body">
|
<div class="card-body">
|
||||||
{% if connections %}
|
{% if connections %}
|
||||||
<div class="table-responsive">
|
<div class="table-responsive">
|
||||||
<table class="table table-hover">
|
<table class="table table-hover align-middle">
|
||||||
<thead>
|
<thead>
|
||||||
<tr>
|
<tr>
|
||||||
<th>Name</th>
|
<th>Name</th>
|
||||||
<th>Endpoint</th>
|
<th>Endpoint</th>
|
||||||
<th>Region</th>
|
<th>Region</th>
|
||||||
<th>Access Key</th>
|
<th>Access Key</th>
|
||||||
<th>Actions</th>
|
<th class="text-end">Actions</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{% for conn in connections %}
|
{% for conn in connections %}
|
||||||
<tr>
|
<tr>
|
||||||
<td>{{ conn.name }}</td>
|
<td class="fw-medium">{{ conn.name }}</td>
|
||||||
<td>{{ conn.endpoint_url }}</td>
|
<td class="small text-muted">{{ conn.endpoint_url }}</td>
|
||||||
<td>{{ conn.region }}</td>
|
<td><span class="badge bg-light text-dark border">{{ conn.region }}</span></td>
|
||||||
<td><code>{{ conn.access_key }}</code></td>
|
<td><code class="small">{{ conn.access_key }}</code></td>
|
||||||
<td>
|
<td class="text-end">
|
||||||
<form method="POST" action="{{ url_for('ui.delete_connection', connection_id=conn.id) }}" onsubmit="return confirm('Are you sure?');" style="display: inline;">
|
<div class="btn-group">
|
||||||
<button type="submit" class="btn btn-sm btn-danger">Delete</button>
|
<button type="button" class="btn btn-sm btn-outline-primary"
|
||||||
</form>
|
data-bs-toggle="modal"
|
||||||
|
data-bs-target="#editConnectionModal"
|
||||||
|
data-id="{{ conn.id }}"
|
||||||
|
data-name="{{ conn.name }}"
|
||||||
|
data-endpoint="{{ conn.endpoint_url }}"
|
||||||
|
data-region="{{ conn.region }}"
|
||||||
|
data-access="{{ conn.access_key }}"
|
||||||
|
data-secret="{{ conn.secret_key }}">
|
||||||
|
Edit
|
||||||
|
</button>
|
||||||
|
<button type="button" class="btn btn-sm btn-outline-danger"
|
||||||
|
data-bs-toggle="modal"
|
||||||
|
data-bs-target="#deleteConnectionModal"
|
||||||
|
data-id="{{ conn.id }}"
|
||||||
|
data-name="{{ conn.name }}">
|
||||||
|
Delete
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
@@ -80,10 +107,164 @@
|
|||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
{% else %}
|
{% else %}
|
||||||
<p class="text-muted text-center my-4">No remote connections configured.</p>
|
<div class="text-center py-5 text-muted">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="48" height="48" fill="currentColor" class="bi bi-hdd-network mb-3" viewBox="0 0 16 16">
|
||||||
|
<path d="M4.5 5a.5.5 0 1 0 0-1 .5.5 0 0 0 0 1zM3 4.5a.5.5 0 1 1-1 0 .5.5 0 0 1 1 0z"/>
|
||||||
|
<path d="M0 4a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v1a2 2 0 0 1-2 2H8.5v3a1.5 1.5 0 0 1 1.5 1.5v3.375a1.125 1.125 0 0 1-1.125 1.125h-1.75a1.125 1.125 0 0 1-1.125-1.125V11.5A1.5 1.5 0 0 1 7.5 10V7H2a2 2 0 0 1-2-2V4zm1 0v1a1 1 0 0 0 1 1h12a1 1 0 0 0 1-1V4a1 1 0 0 0-1-1H2a1 1 0 0 0-1 1zm6 5v1.5a.5.5 0 0 0 .5.5h1.75a.5.5 0 0 0 .5-.5V10a.5.5 0 0 0-.5-.5H7.5a.5.5 0 0 0-.5.5z"/>
|
||||||
|
</svg>
|
||||||
|
<p>No remote connections configured.</p>
|
||||||
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Edit Connection Modal -->
|
||||||
|
<div class="modal fade" id="editConnectionModal" tabindex="-1" aria-hidden="true">
|
||||||
|
<div class="modal-dialog">
|
||||||
|
<div class="modal-content">
|
||||||
|
<div class="modal-header">
|
||||||
|
<h5 class="modal-title">Edit Connection</h5>
|
||||||
|
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||||
|
</div>
|
||||||
|
<form method="POST" id="editConnectionForm">
|
||||||
|
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||||
|
<div class="modal-body">
|
||||||
|
<div class="mb-3">
|
||||||
|
<label for="edit_name" class="form-label">Name</label>
|
||||||
|
<input type="text" class="form-control" id="edit_name" name="name" required>
|
||||||
|
</div>
|
||||||
|
<div class="mb-3">
|
||||||
|
<label for="edit_endpoint_url" class="form-label">Endpoint URL</label>
|
||||||
|
<input type="url" class="form-control" id="edit_endpoint_url" name="endpoint_url" required>
|
||||||
|
</div>
|
||||||
|
<div class="mb-3">
|
||||||
|
<label for="edit_region" class="form-label">Region</label>
|
||||||
|
<input type="text" class="form-control" id="edit_region" name="region" required>
|
||||||
|
</div>
|
||||||
|
<div class="mb-3">
|
||||||
|
<label for="edit_access_key" class="form-label">Access Key</label>
|
||||||
|
<input type="text" class="form-control" id="edit_access_key" name="access_key" required>
|
||||||
|
</div>
|
||||||
|
<div class="mb-3">
|
||||||
|
<label for="edit_secret_key" class="form-label">Secret Key</label>
|
||||||
|
<div class="input-group">
|
||||||
|
<input type="password" class="form-control" id="edit_secret_key" name="secret_key" required>
|
||||||
|
<button class="btn btn-outline-secondary" type="button" onclick="togglePassword('edit_secret_key')">
|
||||||
|
Show
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div id="editTestResult" class="mt-2"></div>
|
||||||
|
</div>
|
||||||
|
<div class="modal-footer">
|
||||||
|
<button type="button" class="btn btn-outline-secondary" id="editTestConnectionBtn">Test Connection</button>
|
||||||
|
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||||
|
<button type="submit" class="btn btn-primary">Save Changes</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Delete Connection Modal -->
|
||||||
|
<div class="modal fade" id="deleteConnectionModal" tabindex="-1" aria-hidden="true">
|
||||||
|
<div class="modal-dialog modal-dialog-centered">
|
||||||
|
<div class="modal-content">
|
||||||
|
<div class="modal-header">
|
||||||
|
<h5 class="modal-title">Delete Connection</h5>
|
||||||
|
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||||
|
</div>
|
||||||
|
<div class="modal-body">
|
||||||
|
<p>Are you sure you want to delete the connection <strong id="deleteConnectionName"></strong>?</p>
|
||||||
|
<p class="text-muted small">This will stop any replication rules using this connection.</p>
|
||||||
|
</div>
|
||||||
|
<div class="modal-footer">
|
||||||
|
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||||
|
<form method="POST" id="deleteConnectionForm">
|
||||||
|
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||||
|
<button type="submit" class="btn btn-danger">Delete</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
function togglePassword(id) {
|
||||||
|
const input = document.getElementById(id);
|
||||||
|
if (input.type === "password") {
|
||||||
|
input.type = "text";
|
||||||
|
} else {
|
||||||
|
input.type = "password";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test Connection Logic
|
||||||
|
async function testConnection(formId, resultId) {
|
||||||
|
const form = document.getElementById(formId);
|
||||||
|
const resultDiv = document.getElementById(resultId);
|
||||||
|
const formData = new FormData(form);
|
||||||
|
const data = Object.fromEntries(formData.entries());
|
||||||
|
|
||||||
|
resultDiv.innerHTML = '<div class="text-info"><span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span> Testing...</div>';
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch("{{ url_for('ui.test_connection') }}", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"X-CSRFToken": "{{ csrf_token() }}"
|
||||||
|
},
|
||||||
|
body: JSON.stringify(data)
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await response.json();
|
||||||
|
if (response.ok) {
|
||||||
|
resultDiv.innerHTML = `<div class="text-success"><i class="bi bi-check-circle"></i> ${result.message}</div>`;
|
||||||
|
} else {
|
||||||
|
resultDiv.innerHTML = `<div class="text-danger"><i class="bi bi-exclamation-circle"></i> ${result.message}</div>`;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
resultDiv.innerHTML = `<div class="text-danger"><i class="bi bi-exclamation-circle"></i> Connection failed</div>`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
document.getElementById('testConnectionBtn').addEventListener('click', () => {
|
||||||
|
testConnection('createConnectionForm', 'testResult');
|
||||||
|
});
|
||||||
|
|
||||||
|
document.getElementById('editTestConnectionBtn').addEventListener('click', () => {
|
||||||
|
testConnection('editConnectionForm', 'editTestResult');
|
||||||
|
});
|
||||||
|
|
||||||
|
// Modal Event Listeners
|
||||||
|
const editModal = document.getElementById('editConnectionModal');
|
||||||
|
editModal.addEventListener('show.bs.modal', event => {
|
||||||
|
const button = event.relatedTarget;
|
||||||
|
const id = button.getAttribute('data-id');
|
||||||
|
|
||||||
|
document.getElementById('edit_name').value = button.getAttribute('data-name');
|
||||||
|
document.getElementById('edit_endpoint_url').value = button.getAttribute('data-endpoint');
|
||||||
|
document.getElementById('edit_region').value = button.getAttribute('data-region');
|
||||||
|
document.getElementById('edit_access_key').value = button.getAttribute('data-access');
|
||||||
|
document.getElementById('edit_secret_key').value = button.getAttribute('data-secret');
|
||||||
|
document.getElementById('editTestResult').innerHTML = '';
|
||||||
|
|
||||||
|
const form = document.getElementById('editConnectionForm');
|
||||||
|
form.action = "{{ url_for('ui.update_connection', connection_id='CONN_ID') }}".replace('CONN_ID', id);
|
||||||
|
});
|
||||||
|
|
||||||
|
const deleteModal = document.getElementById('deleteConnectionModal');
|
||||||
|
deleteModal.addEventListener('show.bs.modal', event => {
|
||||||
|
const button = event.relatedTarget;
|
||||||
|
const id = button.getAttribute('data-id');
|
||||||
|
const name = button.getAttribute('data-name');
|
||||||
|
|
||||||
|
document.getElementById('deleteConnectionName').textContent = name;
|
||||||
|
const form = document.getElementById('deleteConnectionForm');
|
||||||
|
form.action = "{{ url_for('ui.delete_connection', connection_id='CONN_ID') }}".replace('CONN_ID', id);
|
||||||
|
});
|
||||||
|
</script>
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|||||||
@@ -290,6 +290,18 @@ s3.complete_multipart_upload(
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<h3 class="h6 text-uppercase text-muted mt-4">Bidirectional Replication (Active-Active)</h3>
|
||||||
|
<p class="small text-muted">To set up two-way replication (Server A ↔ Server B):</p>
|
||||||
|
<ol class="docs-steps mb-3">
|
||||||
|
<li>Follow the steps above to replicate <strong>A → B</strong>.</li>
|
||||||
|
<li>Repeat the process on Server B to replicate <strong>B → A</strong> (create a connection to A, enable rule).</li>
|
||||||
|
</ol>
|
||||||
|
<p class="small text-muted mb-0">
|
||||||
|
<strong>Loop Prevention:</strong> The system automatically detects replication traffic using a custom User-Agent (<code>S3ReplicationAgent</code>). This prevents infinite loops where an object replicated from A to B is immediately replicated back to A.
|
||||||
|
<br>
|
||||||
|
<strong>Deletes:</strong> Deleting an object on one server will propagate the deletion to the other server.
|
||||||
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</article>
|
</article>
|
||||||
<article id="troubleshooting" class="card shadow-sm docs-section">
|
<article id="troubleshooting" class="card shadow-sm docs-section">
|
||||||
@@ -330,8 +342,8 @@ s3.complete_multipart_upload(
|
|||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td>Requests hit the wrong host</td>
|
<td>Requests hit the wrong host</td>
|
||||||
<td><code>API_BASE_URL</code> not updated after tunneling/forwarding</td>
|
<td>Proxy headers missing or <code>API_BASE_URL</code> incorrect</td>
|
||||||
<td>Set <code>API_BASE_URL</code> in your shell or <code>.env</code> to match the published host.</td>
|
<td>Ensure your proxy sends <code>X-Forwarded-Host</code>/<code>Proto</code> headers, or explicitly set <code>API_BASE_URL</code> to your public domain.</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -286,9 +286,6 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="modal-body" id="rotateSecretConfirm">
|
<div class="modal-body" id="rotateSecretConfirm">
|
||||||
<p>Are you sure you want to rotate the secret key for <strong id="rotateUserLabel"></strong>?</p>
|
<p>Are you sure you want to rotate the secret key for <strong id="rotateUserLabel"></strong>?</p>
|
||||||
<div id="rotateSelfWarning" class="alert alert-warning d-none">
|
|
||||||
<strong>Warning:</strong> You are rotating your own secret key. You will need to sign in again with the new key.
|
|
||||||
</div>
|
|
||||||
<div class="alert alert-warning mb-0">
|
<div class="alert alert-warning mb-0">
|
||||||
The old secret key will stop working immediately. Any applications using it must be updated.
|
The old secret key will stop working immediately. Any applications using it must be updated.
|
||||||
</div>
|
</div>
|
||||||
@@ -474,7 +471,6 @@
|
|||||||
const rotateSecretResult = document.getElementById('rotateSecretResult');
|
const rotateSecretResult = document.getElementById('rotateSecretResult');
|
||||||
const newSecretKeyInput = document.getElementById('newSecretKey');
|
const newSecretKeyInput = document.getElementById('newSecretKey');
|
||||||
const copyNewSecretBtn = document.getElementById('copyNewSecret');
|
const copyNewSecretBtn = document.getElementById('copyNewSecret');
|
||||||
const rotateSelfWarning = document.getElementById('rotateSelfWarning');
|
|
||||||
let currentRotateKey = null;
|
let currentRotateKey = null;
|
||||||
|
|
||||||
document.querySelectorAll('[data-rotate-user]').forEach(btn => {
|
document.querySelectorAll('[data-rotate-user]').forEach(btn => {
|
||||||
@@ -482,12 +478,6 @@
|
|||||||
currentRotateKey = btn.dataset.rotateUser;
|
currentRotateKey = btn.dataset.rotateUser;
|
||||||
rotateUserLabel.textContent = currentRotateKey;
|
rotateUserLabel.textContent = currentRotateKey;
|
||||||
|
|
||||||
if (currentRotateKey === currentUserKey) {
|
|
||||||
rotateSelfWarning.classList.remove('d-none');
|
|
||||||
} else {
|
|
||||||
rotateSelfWarning.classList.add('d-none');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reset Modal State
|
// Reset Modal State
|
||||||
rotateSecretConfirm.classList.remove('d-none');
|
rotateSecretConfirm.classList.remove('d-none');
|
||||||
rotateSecretResult.classList.add('d-none');
|
rotateSecretResult.classList.add('d-none');
|
||||||
|
|||||||
Reference in New Issue
Block a user