diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ba575da --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.git +.gitignore +.venv +__pycache__ +*.pyc +*.pyo +*.pyd +.pytest_cache +.coverage +htmlcov +logs +data +tmp diff --git a/Dockerfile b/Dockerfile index 6c0c97e..b18c77c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,9 +16,14 @@ RUN pip install --no-cache-dir -r requirements.txt COPY . . -# Drop privileges -RUN useradd -m -u 1000 myfsio \ +# Make entrypoint executable +RUN chmod +x docker-entrypoint.sh + +# Create data directory and set permissions +RUN mkdir -p /app/data \ + && useradd -m -u 1000 myfsio \ && chown -R myfsio:myfsio /app + USER myfsio EXPOSE 5000 5100 @@ -29,4 +34,4 @@ ENV APP_HOST=0.0.0.0 \ HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ CMD python -c "import requests; requests.get('http://localhost:5000/healthz', timeout=2)" -CMD ["python", "run.py", "--mode", "both"] +CMD ["./docker-entrypoint.sh"] diff --git a/app/__init__.py b/app/__init__.py index 17da538..3ea9264 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging +import sys import time import uuid from logging.handlers import RotatingFileHandler @@ -12,6 +13,7 @@ from typing import Any, Dict, Optional from flask import Flask, g, has_request_context, redirect, render_template, request, url_for from flask_cors import CORS from flask_wtf.csrf import CSRFError +from werkzeug.middleware.proxy_fix import ProxyFix from .bucket_policies import BucketPolicyStore from .config import AppConfig @@ -47,6 +49,9 @@ def create_app( if app.config.get("TESTING"): app.config.setdefault("WTF_CSRF_ENABLED", False) + # Trust X-Forwarded-* headers from proxies + app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1) + _configure_cors(app) _configure_logging(app) @@ -167,23 +172,33 @@ class _RequestContextFilter(logging.Filter): def _configure_logging(app: Flask) -> None: - log_file = Path(app.config["LOG_FILE"]) - log_file.parent.mkdir(parents=True, exist_ok=True) - handler = RotatingFileHandler( - log_file, - maxBytes=int(app.config.get("LOG_MAX_BYTES", 5 * 1024 * 1024)), - backupCount=int(app.config.get("LOG_BACKUP_COUNT", 3)), - encoding="utf-8", - ) formatter = logging.Formatter( "%(asctime)s | %(levelname)s | %(request_id)s | %(method)s %(path)s | %(message)s" ) - handler.setFormatter(formatter) - handler.addFilter(_RequestContextFilter()) + + # Stream Handler (stdout) - Primary for Docker + stream_handler = logging.StreamHandler(sys.stdout) + stream_handler.setFormatter(formatter) + stream_handler.addFilter(_RequestContextFilter()) logger = app.logger logger.handlers.clear() - logger.addHandler(handler) + logger.addHandler(stream_handler) + + # File Handler (optional, if configured) + if app.config.get("LOG_TO_FILE"): + log_file = Path(app.config["LOG_FILE"]) + log_file.parent.mkdir(parents=True, exist_ok=True) + file_handler = RotatingFileHandler( + log_file, + maxBytes=int(app.config.get("LOG_MAX_BYTES", 5 * 1024 * 1024)), + backupCount=int(app.config.get("LOG_BACKUP_COUNT", 3)), + encoding="utf-8", + ) + file_handler.setFormatter(formatter) + file_handler.addFilter(_RequestContextFilter()) + logger.addHandler(file_handler) + logger.setLevel(getattr(logging, app.config.get("LOG_LEVEL", "INFO"), logging.INFO)) @app.before_request @@ -211,5 +226,4 @@ def _configure_logging(app: Flask) -> None: }, ) response.headers["X-Request-Duration-ms"] = f"{duration_ms:.2f}" - response.headers["Server"] = "MyFISO" return response diff --git a/app/config.py b/app/config.py index 3bfd14f..2033cbd 100644 --- a/app/config.py +++ b/app/config.py @@ -39,7 +39,7 @@ class AppConfig: secret_key: str iam_config_path: Path bucket_policy_path: Path - api_base_url: str + api_base_url: Optional[str] aws_region: str aws_service: str ui_enforce_bucket_policies: bool @@ -78,11 +78,25 @@ class AppConfig: multipart_min_part_size = int(_get("MULTIPART_MIN_PART_SIZE", 5 * 1024 * 1024)) default_secret = "dev-secret-key" secret_key = str(_get("SECRET_KEY", default_secret)) + + # If using default/missing secret, try to load/persist a generated one from disk + # This ensures consistency across Gunicorn workers if not secret_key or secret_key == default_secret: - generated = secrets.token_urlsafe(32) - if secret_key == default_secret: - warnings.warn("Using insecure default SECRET_KEY. A random value has been generated; set SECRET_KEY for production", RuntimeWarning) - secret_key = generated + secret_file = storage_root / ".myfsio.sys" / "config" / ".secret" + if secret_file.exists(): + secret_key = secret_file.read_text().strip() + else: + generated = secrets.token_urlsafe(32) + if secret_key == default_secret: + warnings.warn("Using insecure default SECRET_KEY. A random value has been generated and persisted; set SECRET_KEY for production", RuntimeWarning) + try: + secret_file.parent.mkdir(parents=True, exist_ok=True) + secret_file.write_text(generated) + secret_key = generated + except OSError: + # Fallback if we can't write to disk (e.g. read-only fs) + secret_key = generated + iam_env_override = "IAM_CONFIG" in overrides or "IAM_CONFIG" in os.environ bucket_policy_override = "BUCKET_POLICY_PATH" in overrides or "BUCKET_POLICY_PATH" in os.environ @@ -100,7 +114,10 @@ class AppConfig: bucket_policy_path, legacy_path=None if bucket_policy_override else PROJECT_ROOT / "data" / "bucket_policies.json", ) - api_base_url = str(_get("API_BASE_URL", "http://127.0.0.1:5000")) + api_base_url = _get("API_BASE_URL", None) + if api_base_url: + api_base_url = str(api_base_url) + aws_region = str(_get("AWS_REGION", "us-east-1")) aws_service = str(_get("AWS_SERVICE", "s3")) enforce_ui_policies = str(_get("UI_ENFORCE_BUCKET_POLICIES", "0")).lower() in {"1", "true", "yes", "on"} diff --git a/app/iam.py b/app/iam.py index f6dc33c..3ab64c2 100644 --- a/app/iam.py +++ b/app/iam.py @@ -77,10 +77,20 @@ class IamService: self._users: Dict[str, Dict[str, Any]] = {} self._raw_config: Dict[str, Any] = {} self._failed_attempts: Dict[str, Deque[datetime]] = {} + self._last_load_time = 0.0 self._load() + def _maybe_reload(self) -> None: + """Reload configuration if the file has changed on disk.""" + try: + if self.config_path.stat().st_mtime > self._last_load_time: + self._load() + except OSError: + pass + # ---------------------- authz helpers ---------------------- def authenticate(self, access_key: str, secret_key: str) -> Principal: + self._maybe_reload() access_key = (access_key or "").strip() secret_key = (secret_key or "").strip() if not access_key or not secret_key: @@ -135,12 +145,14 @@ class IamService: return int(max(0, self.auth_lockout_window.total_seconds() - elapsed)) def principal_for_key(self, access_key: str) -> Principal: + self._maybe_reload() record = self._users.get(access_key) if not record: raise IamError("Unknown access key") return self._build_principal(access_key, record) def secret_for_key(self, access_key: str) -> str: + self._maybe_reload() record = self._users.get(access_key) if not record: raise IamError("Unknown access key") @@ -245,6 +257,7 @@ class IamService: # ---------------------- config helpers ---------------------- def _load(self) -> None: try: + self._last_load_time = self.config_path.stat().st_mtime content = self.config_path.read_text(encoding='utf-8') raw = json.loads(content) except FileNotFoundError: diff --git a/app/replication.py b/app/replication.py index b9d86ee..c604727 100644 --- a/app/replication.py +++ b/app/replication.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging +import mimetypes import threading from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass @@ -9,13 +10,17 @@ from pathlib import Path from typing import Dict, Optional import boto3 +from botocore.config import Config from botocore.exceptions import ClientError +from boto3.exceptions import S3UploadFailedError from .connections import ConnectionStore, RemoteConnection -from .storage import ObjectStorage +from .storage import ObjectStorage, StorageError logger = logging.getLogger(__name__) +REPLICATION_USER_AGENT = "S3ReplicationAgent/1.0" + @dataclass class ReplicationRule: @@ -66,7 +71,26 @@ class ReplicationManager: del self._rules[bucket_name] self.save_rules() - def trigger_replication(self, bucket_name: str, object_key: str) -> None: + def create_remote_bucket(self, connection_id: str, bucket_name: str) -> None: + """Create a bucket on the remote connection.""" + connection = self.connections.get(connection_id) + if not connection: + raise ValueError(f"Connection {connection_id} not found") + + try: + s3 = boto3.client( + "s3", + endpoint_url=connection.endpoint_url, + aws_access_key_id=connection.access_key, + aws_secret_access_key=connection.secret_key, + region_name=connection.region, + ) + s3.create_bucket(Bucket=bucket_name) + except ClientError as e: + logger.error(f"Failed to create remote bucket {bucket_name}: {e}") + raise + + def trigger_replication(self, bucket_name: str, object_key: str, action: str = "write") -> None: rule = self.get_rule(bucket_name) if not rule or not rule.enabled: return @@ -76,42 +100,93 @@ class ReplicationManager: logger.warning(f"Replication skipped for {bucket_name}/{object_key}: Connection {rule.target_connection_id} not found") return - self._executor.submit(self._replicate_task, bucket_name, object_key, rule, connection) + self._executor.submit(self._replicate_task, bucket_name, object_key, rule, connection, action) - def _replicate_task(self, bucket_name: str, object_key: str, rule: ReplicationRule, conn: RemoteConnection) -> None: + def _replicate_task(self, bucket_name: str, object_key: str, rule: ReplicationRule, conn: RemoteConnection, action: str) -> None: try: - # 1. Get local file path - # Note: We are accessing internal storage structure here. - # Ideally storage.py should expose a 'get_file_path' or we read the stream. - # For efficiency, we'll try to read the file directly if we can, or use storage.get_object - # Using boto3 to upload + config = Config(user_agent_extra=REPLICATION_USER_AGENT) s3 = boto3.client( "s3", endpoint_url=conn.endpoint_url, aws_access_key_id=conn.access_key, aws_secret_access_key=conn.secret_key, region_name=conn.region, + config=config, ) + if action == "delete": + try: + s3.delete_object(Bucket=rule.target_bucket, Key=object_key) + logger.info(f"Replicated DELETE {bucket_name}/{object_key} to {conn.name} ({rule.target_bucket})") + except ClientError as e: + logger.error(f"Replication DELETE failed for {bucket_name}/{object_key}: {e}") + return + + # 1. Get local file path + # Note: We are accessing internal storage structure here. + # Ideally storage.py should expose a 'get_file_path' or we read the stream. + # For efficiency, we'll try to read the file directly if we can, or use storage.get_object + # We need the file content. # Since ObjectStorage is filesystem based, let's get the stream. # We need to be careful about closing it. - meta = self.storage.get_object_meta(bucket_name, object_key) - if not meta: + try: + path = self.storage.get_object_path(bucket_name, object_key) + except StorageError: return - with self.storage.open_object(bucket_name, object_key) as f: - extra_args = {} - if meta.metadata: - extra_args["Metadata"] = meta.metadata - - s3.upload_fileobj( - f, - rule.target_bucket, - object_key, - ExtraArgs=extra_args - ) + metadata = self.storage.get_object_metadata(bucket_name, object_key) + + extra_args = {} + if metadata: + extra_args["Metadata"] = metadata + + # Guess content type to prevent corruption/wrong handling + content_type, _ = mimetypes.guess_type(path) + file_size = path.stat().st_size + + logger.info(f"Replicating {bucket_name}/{object_key}: Size={file_size}, ContentType={content_type}") + + try: + with path.open("rb") as f: + s3.put_object( + Bucket=rule.target_bucket, + Key=object_key, + Body=f, + ContentLength=file_size, + ContentType=content_type or "application/octet-stream", + Metadata=metadata or {} + ) + except (ClientError, S3UploadFailedError) as e: + # Check if it's a NoSuchBucket error (either direct or wrapped) + is_no_bucket = False + if isinstance(e, ClientError): + if e.response['Error']['Code'] == 'NoSuchBucket': + is_no_bucket = True + elif isinstance(e, S3UploadFailedError): + if "NoSuchBucket" in str(e): + is_no_bucket = True + + if is_no_bucket: + logger.info(f"Target bucket {rule.target_bucket} not found. Attempting to create it.") + try: + s3.create_bucket(Bucket=rule.target_bucket) + # Retry upload + with path.open("rb") as f: + s3.put_object( + Bucket=rule.target_bucket, + Key=object_key, + Body=f, + ContentLength=file_size, + ContentType=content_type or "application/octet-stream", + Metadata=metadata or {} + ) + except Exception as create_err: + logger.error(f"Failed to create target bucket {rule.target_bucket}: {create_err}") + raise e # Raise original error + else: + raise e logger.info(f"Replicated {bucket_name}/{object_key} to {conn.name} ({rule.target_bucket})") diff --git a/app/s3_api.py b/app/s3_api.py index 584074b..b080346 100644 --- a/app/s3_api.py +++ b/app/s3_api.py @@ -8,7 +8,7 @@ import re import uuid from datetime import datetime, timedelta, timezone from typing import Any, Dict -from urllib.parse import quote, urlencode +from urllib.parse import quote, urlencode, urlparse from xml.etree.ElementTree import Element, SubElement, tostring, fromstring, ParseError from flask import Blueprint, Response, current_app, jsonify, request @@ -17,6 +17,7 @@ from werkzeug.http import http_date from .bucket_policies import BucketPolicyStore from .extensions import limiter from .iam import IamError, Principal +from .replication import ReplicationManager from .storage import ObjectStorage, StorageError s3_api_bp = Blueprint("s3_api", __name__) @@ -31,6 +32,9 @@ def _iam(): return current_app.extensions["iam"] +def _replication_manager() -> ReplicationManager: + return current_app.extensions["replication"] + def _bucket_policies() -> BucketPolicyStore: store: BucketPolicyStore = current_app.extensions["bucket_policies"] @@ -405,7 +409,11 @@ def _canonical_headers_from_request(headers: list[str]) -> str: lines = [] for header in headers: if header == "host": - value = request.host + api_base = current_app.config.get("API_BASE_URL") + if api_base: + value = urlparse(api_base).netloc + else: + value = request.host else: value = request.headers.get(header, "") canonical_value = " ".join(value.strip().split()) if value else "" @@ -468,7 +476,17 @@ def _generate_presigned_url( "X-Amz-Content-Sha256": "UNSIGNED-PAYLOAD", } canonical_query = _encode_query_params(query_params) - host = request.host + + # Determine host and scheme from config or request + api_base = current_app.config.get("API_BASE_URL") + if api_base: + parsed = urlparse(api_base) + host = parsed.netloc + scheme = parsed.scheme + else: + host = request.headers.get("X-Forwarded-Host", request.host) + scheme = request.headers.get("X-Forwarded-Proto", request.scheme or "http") + canonical_headers = f"host:{host}\n" canonical_request = "\n".join( [ @@ -492,7 +510,6 @@ def _generate_presigned_url( signing_key = _derive_signing_key(secret_key, date_stamp, region, service) signature = hmac.new(signing_key, string_to_sign.encode(), hashlib.sha256).hexdigest() query_with_sig = canonical_query + f"&X-Amz-Signature={signature}" - scheme = request.scheme or "http" return f"{scheme}://{host}{_canonical_uri(bucket_name, object_key)}?{query_with_sig}" @@ -1026,6 +1043,7 @@ def bucket_handler(bucket_name: str) -> Response: try: storage.delete_bucket(bucket_name) _bucket_policies().delete_policy(bucket_name) + _replication_manager().delete_rule(bucket_name) except StorageError as exc: code = "BucketNotEmpty" if "not empty" in str(exc) else "NoSuchBucket" status = 409 if code == "BucketNotEmpty" else 404 @@ -1069,7 +1087,12 @@ def object_handler(bucket_name: str, object_key: str): _, error = _object_principal("write", bucket_name, object_key) if error: return error + stream = request.stream + content_encoding = request.headers.get("Content-Encoding", "").lower() + if "aws-chunked" in content_encoding: + stream = AwsChunkedDecoder(stream) + metadata = _extract_request_metadata() try: meta = storage.put_object( @@ -1089,6 +1112,12 @@ def object_handler(bucket_name: str, object_key: str): ) response = Response(status=200) response.headers["ETag"] = f'"{meta.etag}"' + + # Trigger replication if not a replication request + user_agent = request.headers.get("User-Agent", "") + if "S3ReplicationAgent" not in user_agent: + _replication_manager().trigger_replication(bucket_name, object_key, action="write") + return response if request.method in {"GET", "HEAD"}: @@ -1123,6 +1152,12 @@ def object_handler(bucket_name: str, object_key: str): return error storage.delete_object(bucket_name, object_key) current_app.logger.info("Object deleted", extra={"bucket": bucket_name, "key": object_key}) + + # Trigger replication if not a replication request + user_agent = request.headers.get("User-Agent", "") + if "S3ReplicationAgent" not in user_agent: + _replication_manager().trigger_replication(bucket_name, object_key, action="delete") + return Response(status=204) @@ -1243,3 +1278,79 @@ def head_object(bucket_name: str, object_key: str) -> Response: return _error_response("NoSuchKey", "Object not found", 404) except IamError as exc: return _error_response("AccessDenied", str(exc), 403) + + +class AwsChunkedDecoder: + """Decodes aws-chunked encoded streams.""" + def __init__(self, stream): + self.stream = stream + self.buffer = b"" + self.chunk_remaining = 0 + self.finished = False + + def read(self, size=-1): + if self.finished: + return b"" + + result = b"" + while size == -1 or len(result) < size: + if self.chunk_remaining > 0: + to_read = self.chunk_remaining + if size != -1: + to_read = min(to_read, size - len(result)) + + chunk = self.stream.read(to_read) + if not chunk: + raise IOError("Unexpected EOF in chunk data") + + result += chunk + self.chunk_remaining -= len(chunk) + + if self.chunk_remaining == 0: + # Read CRLF after chunk data + crlf = self.stream.read(2) + if crlf != b"\r\n": + raise IOError("Malformed chunk: missing CRLF") + else: + # Read chunk size line + line = b"" + while True: + char = self.stream.read(1) + if not char: + if not line: # EOF at start of chunk size + self.finished = True + return result + raise IOError("Unexpected EOF in chunk size") + line += char + if line.endswith(b"\r\n"): + break + + # Parse chunk size (hex) + try: + line_str = line.decode("ascii").strip() + # Handle chunk-signature extension if present (e.g. "1000;chunk-signature=...") + if ";" in line_str: + line_str = line_str.split(";")[0] + chunk_size = int(line_str, 16) + except ValueError: + raise IOError(f"Invalid chunk size: {line}") + + if chunk_size == 0: + self.finished = True + # Read trailers if any (until empty line) + while True: + line = b"" + while True: + char = self.stream.read(1) + if not char: + break + line += char + if line.endswith(b"\r\n"): + break + if line == b"\r\n" or not line: + break + return result + + self.chunk_remaining = chunk_size + + return result diff --git a/app/ui.py b/app/ui.py index cdc587c..adf1f6f 100644 --- a/app/ui.py +++ b/app/ui.py @@ -6,7 +6,9 @@ import uuid from typing import Any from urllib.parse import urlparse +import boto3 import requests +from botocore.exceptions import ClientError from flask import ( Blueprint, Response, @@ -38,6 +40,10 @@ def _storage() -> ObjectStorage: return current_app.extensions["object_storage"] +def _replication_manager() -> ReplicationManager: + return current_app.extensions["replication"] + + def _iam(): return current_app.extensions["iam"] @@ -494,6 +500,7 @@ def delete_bucket(bucket_name: str): _authorize_ui(principal, bucket_name, "delete") _storage().delete_bucket(bucket_name) _bucket_policies().delete_policy(bucket_name) + _replication_manager().delete_rule(bucket_name) flash(f"Bucket '{bucket_name}' removed", "success") except (StorageError, IamError) as exc: flash(_friendly_error_message(exc), "danger") @@ -512,6 +519,7 @@ def delete_object(bucket_name: str, object_key: str): flash(f"Permanently deleted '{object_key}' and all versions", "success") else: _storage().delete_object(bucket_name, object_key) + _replication_manager().trigger_replication(bucket_name, object_key, action="delete") flash(f"Deleted '{object_key}'", "success") except (IamError, StorageError) as exc: flash(_friendly_error_message(exc), "danger") @@ -572,6 +580,7 @@ def bulk_delete_objects(bucket_name: str): storage.purge_object(bucket_name, key) else: storage.delete_object(bucket_name, key) + _replication_manager().trigger_replication(bucket_name, key, action="delete") deleted.append(key) except StorageError as exc: errors.append({"key": key, "error": str(exc)}) @@ -701,10 +710,21 @@ def object_presign(bucket_name: str, object_key: str): _authorize_ui(principal, bucket_name, action, object_key=object_key) except IamError as exc: return jsonify({"error": str(exc)}), 403 - api_base = current_app.config["API_BASE_URL"].rstrip("/") + + api_base = current_app.config.get("API_BASE_URL") + if not api_base: + api_base = "http://127.0.0.1:5000" + api_base = api_base.rstrip("/") + url = f"{api_base}/presign/{bucket_name}/{object_key}" + + headers = _api_headers() + # Forward the host so the API knows the public URL + headers["X-Forwarded-Host"] = request.host + headers["X-Forwarded-Proto"] = request.scheme + try: - response = requests.post(url, headers=_api_headers(), json=payload, timeout=5) + response = requests.post(url, headers=headers, json=payload, timeout=5) except requests.RequestException as exc: return jsonify({"error": f"API unavailable: {exc}"}), 502 try: @@ -926,6 +946,12 @@ def rotate_iam_secret(access_key: str): return redirect(url_for("ui.iam_dashboard")) try: new_secret = _iam().rotate_secret(access_key) + # If rotating own key, update session immediately so subsequent API calls (like presign) work + if principal and principal.access_key == access_key: + creds = session.get("credentials", {}) + creds["secret_key"] = new_secret + session["credentials"] = creds + session.modified = True except IamError as exc: if request.accept_mimetypes.accept_json and not request.accept_mimetypes.accept_html: return jsonify({"error": str(exc)}), 400 @@ -1064,6 +1090,73 @@ def create_connection(): return redirect(url_for("ui.connections_dashboard")) +@ui_bp.post("/connections/test") +def test_connection(): + principal = _current_principal() + try: + _iam().authorize(principal, None, "iam:list_users") + except IamError: + return jsonify({"status": "error", "message": "Access denied"}), 403 + + data = request.get_json(silent=True) or request.form + endpoint = data.get("endpoint_url", "").strip() + access_key = data.get("access_key", "").strip() + secret_key = data.get("secret_key", "").strip() + region = data.get("region", "us-east-1").strip() + + if not all([endpoint, access_key, secret_key]): + return jsonify({"status": "error", "message": "Missing credentials"}), 400 + + try: + s3 = boto3.client( + "s3", + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=region, + ) + # Try to list buckets to verify credentials and endpoint + s3.list_buckets() + return jsonify({"status": "ok", "message": "Connection successful"}) + except Exception as e: + return jsonify({"status": "error", "message": str(e)}), 400 + + +@ui_bp.post("/connections//update") +def update_connection(connection_id: str): + principal = _current_principal() + try: + _iam().authorize(principal, None, "iam:list_users") + except IamError: + flash("Access denied", "danger") + return redirect(url_for("ui.buckets_overview")) + + conn = _connections().get(connection_id) + if not conn: + flash("Connection not found", "danger") + return redirect(url_for("ui.connections_dashboard")) + + name = request.form.get("name", "").strip() + endpoint = request.form.get("endpoint_url", "").strip() + access_key = request.form.get("access_key", "").strip() + secret_key = request.form.get("secret_key", "").strip() + region = request.form.get("region", "us-east-1").strip() + + if not all([name, endpoint, access_key, secret_key]): + flash("All fields are required", "danger") + return redirect(url_for("ui.connections_dashboard")) + + conn.name = name + conn.endpoint_url = endpoint + conn.access_key = access_key + conn.secret_key = secret_key + conn.region = region + + _connections().save() + flash(f"Connection '{name}' updated", "success") + return redirect(url_for("ui.connections_dashboard")) + + @ui_bp.post("/connections//delete") def delete_connection(connection_id: str): principal = _current_principal() diff --git a/app/version.py b/app/version.py index d2eb696..bda6952 100644 --- a/app/version.py +++ b/app/version.py @@ -1,7 +1,7 @@ """Central location for the application version string.""" from __future__ import annotations -APP_VERSION = "0.1.0" +APP_VERSION = "0.1.1" def get_version() -> str: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 0000000..08c3a3a --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -e + +# Run both services using the python runner in production mode +exec python run.py --prod diff --git a/docs.md b/docs.md index e55a737..016ffbb 100644 --- a/docs.md +++ b/docs.md @@ -77,12 +77,20 @@ The repo now tracks a human-friendly release string inside `app/version.py` (see | `SECRET_KEY` | `dev-secret-key` | Flask session key for UI auth. | | `IAM_CONFIG` | `/data/.myfsio.sys/config/iam.json` | Stores users, secrets, and inline policies. | | `BUCKET_POLICY_PATH` | `/data/.myfsio.sys/config/bucket_policies.json` | Bucket policy store (auto hot-reload). | -| `API_BASE_URL` | `http://127.0.0.1:5000` | Used by the UI to hit API endpoints (presign/policy). | +| `API_BASE_URL` | `None` | Used by the UI to hit API endpoints (presign/policy). If unset, the UI will auto-detect the host or use `X-Forwarded-*` headers. | | `AWS_REGION` | `us-east-1` | Region embedded in SigV4 credential scope. | | `AWS_SERVICE` | `s3` | Service string for SigV4. | Set env vars (or pass overrides to `create_app`) to point the servers at custom paths. +### Proxy Configuration + +If running behind a reverse proxy (e.g., Nginx, Cloudflare, or a tunnel), ensure the proxy sets the standard forwarding headers: +- `X-Forwarded-Host` +- `X-Forwarded-Proto` + +The application automatically trusts these headers to generate correct presigned URLs (e.g., `https://s3.example.com/...` instead of `http://127.0.0.1:5000/...`). Alternatively, you can explicitly set `API_BASE_URL` to your public endpoint. + ## 4. Authentication & IAM 1. On first boot, `data/.myfsio.sys/config/iam.json` is seeded with `localadmin / localadmin` that has wildcard access. @@ -262,6 +270,21 @@ Now, configure the primary instance to replicate to the target. aws --endpoint-url http://target-server:5002 s3 ls s3://backup-bucket ``` +### Bidirectional Replication (Active-Active) + +To set up two-way replication (Server A ↔ Server B): + +1. Follow the steps above to replicate **A → B**. +2. Repeat the process on Server B to replicate **B → A**: + - Create a connection on Server B pointing to Server A. + - Enable replication on the target bucket on Server B. + +**Loop Prevention**: The system automatically detects replication traffic using a custom User-Agent (`S3ReplicationAgent`). This prevents infinite loops where an object replicated from A to B is immediately replicated back to A. + +**Deletes**: Deleting an object on one server will propagate the deletion to the other server. + +**Note**: Deleting a bucket will automatically remove its associated replication configuration. + ## 7. Running Tests ```bash diff --git a/requirements.txt b/requirements.txt index 356c1f9..43f1ae7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ Flask-WTF>=1.2.1 pytest>=7.4 requests>=2.31 boto3>=1.34 +waitress>=2.1.2 diff --git a/run.py b/run.py index efd12a2..230ca48 100644 --- a/run.py +++ b/run.py @@ -18,20 +18,28 @@ def _is_debug_enabled() -> bool: return os.getenv("FLASK_DEBUG", "0").lower() in ("1", "true", "yes") -def serve_api(port: int) -> None: +def serve_api(port: int, prod: bool = False) -> None: app = create_api_app() - debug = _is_debug_enabled() - if debug: - warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning) - app.run(host=_server_host(), port=port, debug=debug) + if prod: + from waitress import serve + serve(app, host=_server_host(), port=port, ident="MyFSIO") + else: + debug = _is_debug_enabled() + if debug: + warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning) + app.run(host=_server_host(), port=port, debug=debug) -def serve_ui(port: int) -> None: +def serve_ui(port: int, prod: bool = False) -> None: app = create_ui_app() - debug = _is_debug_enabled() - if debug: - warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning) - app.run(host=_server_host(), port=port, debug=debug) + if prod: + from waitress import serve + serve(app, host=_server_host(), port=port, ident="MyFSIO") + else: + debug = _is_debug_enabled() + if debug: + warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning) + app.run(host=_server_host(), port=port, debug=debug) if __name__ == "__main__": @@ -39,18 +47,19 @@ if __name__ == "__main__": parser.add_argument("--mode", choices=["api", "ui", "both"], default="both") parser.add_argument("--api-port", type=int, default=5000) parser.add_argument("--ui-port", type=int, default=5100) + parser.add_argument("--prod", action="store_true", help="Run in production mode using Waitress") args = parser.parse_args() if args.mode in {"api", "both"}: print(f"Starting API server on port {args.api_port}...") - api_proc = Process(target=serve_api, args=(args.api_port,), daemon=True) + api_proc = Process(target=serve_api, args=(args.api_port, args.prod), daemon=True) api_proc.start() else: api_proc = None if args.mode in {"ui", "both"}: print(f"Starting UI server on port {args.ui_port}...") - serve_ui(args.ui_port) + serve_ui(args.ui_port, args.prod) elif api_proc: try: api_proc.join() diff --git a/templates/bucket_detail.html b/templates/bucket_detail.html index af93fe8..7a2a2a1 100644 --- a/templates/bucket_detail.html +++ b/templates/bucket_detail.html @@ -408,16 +408,16 @@ -
- - -
+ {% else %}

Replication allows you to automatically copy new objects from this bucket to a bucket in another S3-compatible service.

{% if connections %}
+
@@ -434,7 +434,7 @@
-
The bucket on the remote service must already exist.
+
If the target bucket does not exist, it will be created automatically.
@@ -708,6 +708,30 @@
+ + +
{% endblock %} {% block extra_scripts %} diff --git a/templates/connections.html b/templates/connections.html index fbe5175..3a1e603 100644 --- a/templates/connections.html +++ b/templates/connections.html @@ -12,12 +12,13 @@
-
-
+
+
Add New Connection
-
+ +
@@ -36,43 +37,69 @@
- +
+ + +
- +
+ + +
+
-
-
+
+
Existing Connections
{% if connections %}
- +
- + {% for conn in connections %} - - - - - + + + + {% endfor %} @@ -80,10 +107,164 @@
Name Endpoint Region Access KeyActionsActions
{{ conn.name }}{{ conn.endpoint_url }}{{ conn.region }}{{ conn.access_key }} -
- -
+ 		{{ conn.name }}{{ conn.endpoint_url }}{{ conn.region }}{{ conn.access_key }} +
+ + +
{% else %} -

No remote connections configured.

+
+ + + + +

No remote connections configured.

+
{% endif %}
+ + + + + + + + {% endblock %} diff --git a/templates/docs.html b/templates/docs.html index e25c124..6f0dd40 100644 --- a/templates/docs.html +++ b/templates/docs.html @@ -290,6 +290,18 @@ s3.complete_multipart_upload(
+ +

Bidirectional Replication (Active-Active)

+

To set up two-way replication (Server A ↔ Server B):

+
    +
  1. Follow the steps above to replicate A → B.
    2. +
  2. Repeat the process on Server B to replicate B → A (create a connection to A, enable rule).
    3. +
+

+ Loop Prevention: The system automatically detects replication traffic using a custom User-Agent (S3ReplicationAgent). This prevents infinite loops where an object replicated from A to B is immediately replicated back to A. +
+ Deletes: Deleting an object on one server will propagate the deletion to the other server. +

@@ -330,8 +342,8 @@ s3.complete_multipart_upload( Requests hit the wrong host - API_BASE_URL not updated after tunneling/forwarding - Set API_BASE_URL in your shell or .env to match the published host. + Proxy headers missing or API_BASE_URL incorrect + Ensure your proxy sends X-Forwarded-Host/Proto headers, or explicitly set API_BASE_URL to your public domain. diff --git a/templates/iam.html b/templates/iam.html index 0b52091..81b74d8 100644 --- a/templates/iam.html +++ b/templates/iam.html @@ -286,9 +286,6 @@

Are you sure you want to rotate the secret key for ?

-
- Warning: You are rotating your own secret key. You will need to sign in again with the new key. -
The old secret key will stop working immediately. Any applications using it must be updated.
@@ -474,7 +471,6 @@ const rotateSecretResult = document.getElementById('rotateSecretResult'); const newSecretKeyInput = document.getElementById('newSecretKey'); const copyNewSecretBtn = document.getElementById('copyNewSecret'); - const rotateSelfWarning = document.getElementById('rotateSelfWarning'); let currentRotateKey = null; document.querySelectorAll('[data-rotate-user]').forEach(btn => { @@ -482,12 +478,6 @@ currentRotateKey = btn.dataset.rotateUser; rotateUserLabel.textContent = currentRotateKey; - if (currentRotateKey === currentUserKey) { - rotateSelfWarning.classList.remove('d-none'); - } else { - rotateSelfWarning.classList.add('d-none'); - } - // Reset Modal State rotateSecretConfirm.classList.remove('d-none'); rotateSecretResult.classList.add('d-none');