diff --git a/.gitignore b/.gitignore index a8f2958..210bd29 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,10 @@ dist/ *.egg-info/ .eggs/ +# Rust / maturin build artifacts +myfsio_core/target/ +myfsio_core/Cargo.lock + # Local runtime artifacts logs/ *.log diff --git a/Dockerfile b/Dockerfile index 184f240..a50ec8b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,15 +5,27 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ WORKDIR /app -RUN apt-get update \ - && apt-get install -y --no-install-recommends build-essential \ +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential curl \ + && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal \ && rm -rf /var/lib/apt/lists/* +ENV PATH="/root/.cargo/bin:${PATH}" + COPY requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY . . +RUN pip install --no-cache-dir maturin \ + && cd myfsio_core \ + && maturin build --release \ + && pip install target/wheels/*.whl \ + && cd .. \ + && rm -rf myfsio_core/target \ + && pip uninstall -y maturin \ + && rustup self uninstall -y + RUN chmod +x docker-entrypoint.sh RUN mkdir -p /app/data \ diff --git a/app/__init__.py b/app/__init__.py index 7befa1d..eb4a753 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -3,6 +3,7 @@ from __future__ import annotations import html as html_module import logging import mimetypes +import os import shutil import sys import time @@ -93,7 +94,14 @@ def create_app( app.config.setdefault("WTF_CSRF_ENABLED", False) # Trust X-Forwarded-* headers from proxies - app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1) + num_proxies = app.config.get("NUM_TRUSTED_PROXIES", 1) + if num_proxies: + if "NUM_TRUSTED_PROXIES" not in os.environ: + logging.getLogger(__name__).warning( + "NUM_TRUSTED_PROXIES not set, defaulting to 1. " + "Set NUM_TRUSTED_PROXIES=0 if not behind a reverse proxy." + ) + app.wsgi_app = ProxyFix(app.wsgi_app, x_for=num_proxies, x_proto=num_proxies, x_host=num_proxies, x_prefix=num_proxies) # Enable gzip compression for responses (10-20x smaller JSON payloads) if app.config.get("ENABLE_GZIP", True): diff --git a/app/admin_api.py b/app/admin_api.py index 1d6b9ab..f650a3f 100644 --- a/app/admin_api.py +++ b/app/admin_api.py @@ -17,7 +17,7 @@ from .extensions import limiter from .iam import IamError, Principal from .replication import ReplicationManager from .site_registry import PeerSite, SiteInfo, SiteRegistry -from .website_domains import WebsiteDomainStore +from .website_domains import WebsiteDomainStore, normalize_domain, is_valid_domain def _is_safe_url(url: str, allow_internal: bool = False) -> bool: @@ -704,10 +704,12 @@ def create_website_domain(): if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False): return _json_error("InvalidRequest", "Website hosting is not enabled", 400) payload = request.get_json(silent=True) or {} - domain = (payload.get("domain") or "").strip().lower() + domain = normalize_domain(payload.get("domain") or "") bucket = (payload.get("bucket") or "").strip() if not domain: return _json_error("ValidationError", "domain is required", 400) + if not is_valid_domain(domain): + return _json_error("ValidationError", f"Invalid domain: '{domain}'", 400) if not bucket: return _json_error("ValidationError", "bucket is required", 400) storage = _storage() @@ -730,10 +732,11 @@ def get_website_domain(domain: str): return error if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False): return _json_error("InvalidRequest", "Website hosting is not enabled", 400) + domain = normalize_domain(domain) bucket = _website_domains().get_bucket(domain) if not bucket: return _json_error("NotFound", f"No mapping found for domain '{domain}'", 404) - return jsonify({"domain": domain.lower(), "bucket": bucket}) + return jsonify({"domain": domain, "bucket": bucket}) @admin_api_bp.route("/website-domains/", methods=["PUT"]) @@ -744,6 +747,7 @@ def update_website_domain(domain: str): return error if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False): return _json_error("InvalidRequest", "Website hosting is not enabled", 400) + domain = normalize_domain(domain) payload = request.get_json(silent=True) or {} bucket = (payload.get("bucket") or "").strip() if not bucket: @@ -752,9 +756,11 @@ def update_website_domain(domain: str): if not storage.bucket_exists(bucket): return _json_error("NoSuchBucket", f"Bucket '{bucket}' does not exist", 404) store = _website_domains() + if not store.get_bucket(domain): + return _json_error("NotFound", f"No mapping found for domain '{domain}'", 404) store.set_mapping(domain, bucket) logger.info("Website domain mapping updated: %s -> %s", domain, bucket) - return jsonify({"domain": domain.lower(), "bucket": bucket}) + return jsonify({"domain": domain, "bucket": bucket}) @admin_api_bp.route("/website-domains/", methods=["DELETE"]) @@ -765,6 +771,7 @@ def delete_website_domain(domain: str): return error if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False): return _json_error("InvalidRequest", "Website hosting is not enabled", 400) + domain = normalize_domain(domain) if not _website_domains().delete_mapping(domain): return _json_error("NotFound", f"No mapping found for domain '{domain}'", 404) logger.info("Website domain mapping deleted: %s", domain) diff --git a/app/bucket_policies.py b/app/bucket_policies.py index fcb1e41..1ff9eb6 100644 --- a/app/bucket_policies.py +++ b/app/bucket_policies.py @@ -75,7 +75,7 @@ def _evaluate_condition_operator( expected_null = condition_values[0].lower() in ("true", "1", "yes") if condition_values else True return is_null == expected_null - return True + return False ACTION_ALIASES = { "s3:listbucket": "list", diff --git a/app/config.py b/app/config.py index 00023c5..9949d81 100644 --- a/app/config.py +++ b/app/config.py @@ -314,7 +314,7 @@ class AppConfig: site_region = str(_get("SITE_REGION", "us-east-1")) site_priority = int(_get("SITE_PRIORITY", 100)) ratelimit_admin = _validate_rate_limit(str(_get("RATE_LIMIT_ADMIN", "60 per minute"))) - num_trusted_proxies = int(_get("NUM_TRUSTED_PROXIES", 0)) + num_trusted_proxies = int(_get("NUM_TRUSTED_PROXIES", 1)) allowed_redirect_hosts_raw = _get("ALLOWED_REDIRECT_HOSTS", "") allowed_redirect_hosts = [h.strip() for h in str(allowed_redirect_hosts_raw).split(",") if h.strip()] allow_internal_endpoints = str(_get("ALLOW_INTERNAL_ENDPOINTS", "0")).lower() in {"1", "true", "yes", "on"} diff --git a/app/iam.py b/app/iam.py index 9e14ee7..65b705a 100644 --- a/app/iam.py +++ b/app/iam.py @@ -164,9 +164,14 @@ class IamService: self._clear_failed_attempts(access_key) return self._build_principal(access_key, record) + _MAX_LOCKOUT_KEYS = 10000 + def _record_failed_attempt(self, access_key: str) -> None: if not access_key: return + if access_key not in self._failed_attempts and len(self._failed_attempts) >= self._MAX_LOCKOUT_KEYS: + oldest_key = min(self._failed_attempts, key=lambda k: self._failed_attempts[k][0] if self._failed_attempts[k] else datetime.min.replace(tzinfo=timezone.utc)) + del self._failed_attempts[oldest_key] attempts = self._failed_attempts.setdefault(access_key, deque()) self._prune_attempts(attempts) attempts.append(datetime.now(timezone.utc)) diff --git a/app/notifications.py b/app/notifications.py index 6951095..ee03ba8 100644 --- a/app/notifications.py +++ b/app/notifications.py @@ -15,29 +15,23 @@ from typing import Any, Dict, List, Optional from urllib.parse import urlparse import requests +from urllib3.util.connection import create_connection as _urllib3_create_connection -def _is_safe_url(url: str, allow_internal: bool = False) -> bool: - """Check if a URL is safe to make requests to (not internal/private). - - Args: - url: The URL to check. - allow_internal: If True, allows internal/private IP addresses. - Use for self-hosted deployments on internal networks. - """ +def _resolve_and_check_url(url: str, allow_internal: bool = False) -> Optional[str]: try: parsed = urlparse(url) hostname = parsed.hostname if not hostname: - return False + return None cloud_metadata_hosts = { "metadata.google.internal", "169.254.169.254", } if hostname.lower() in cloud_metadata_hosts: - return False + return None if allow_internal: - return True + return hostname blocked_hosts = { "localhost", "127.0.0.1", @@ -46,17 +40,46 @@ def _is_safe_url(url: str, allow_internal: bool = False) -> bool: "[::1]", } if hostname.lower() in blocked_hosts: - return False + return None try: resolved_ip = socket.gethostbyname(hostname) ip = ipaddress.ip_address(resolved_ip) if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved: - return False + return None + return resolved_ip except (socket.gaierror, ValueError): - return False - return True + return None except Exception: - return False + return None + + +def _is_safe_url(url: str, allow_internal: bool = False) -> bool: + return _resolve_and_check_url(url, allow_internal) is not None + + +_dns_pin_lock = threading.Lock() + + +def _pinned_post(url: str, pinned_ip: str, **kwargs: Any) -> requests.Response: + parsed = urlparse(url) + hostname = parsed.hostname or "" + session = requests.Session() + original_create = _urllib3_create_connection + + def _create_pinned(address: Any, *args: Any, **kw: Any) -> Any: + host, req_port = address + if host == hostname: + return original_create((pinned_ip, req_port), *args, **kw) + return original_create(address, *args, **kw) + + import urllib3.util.connection as _conn_mod + with _dns_pin_lock: + _conn_mod.create_connection = _create_pinned + try: + return session.post(url, **kwargs) + finally: + _conn_mod.create_connection = original_create + logger = logging.getLogger(__name__) @@ -344,16 +367,18 @@ class NotificationService: self._queue.task_done() def _send_notification(self, event: NotificationEvent, destination: WebhookDestination) -> None: - if not _is_safe_url(destination.url, allow_internal=self._allow_internal_endpoints): - raise RuntimeError(f"Blocked request to cloud metadata service (SSRF protection): {destination.url}") + resolved_ip = _resolve_and_check_url(destination.url, allow_internal=self._allow_internal_endpoints) + if not resolved_ip: + raise RuntimeError(f"Blocked request (SSRF protection): {destination.url}") payload = event.to_s3_event() headers = {"Content-Type": "application/json", **destination.headers} last_error = None for attempt in range(destination.retry_count): try: - response = requests.post( + response = _pinned_post( destination.url, + resolved_ip, json=payload, headers=headers, timeout=destination.timeout_seconds, diff --git a/app/operation_metrics.py b/app/operation_metrics.py index 67a63c2..3a002e1 100644 --- a/app/operation_metrics.py +++ b/app/operation_metrics.py @@ -2,6 +2,7 @@ from __future__ import annotations import json import logging +import random import threading import time from dataclasses import dataclass, field @@ -9,6 +10,8 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, List, Optional +MAX_LATENCY_SAMPLES = 5000 + logger = logging.getLogger(__name__) @@ -22,6 +25,17 @@ class OperationStats: latency_max_ms: float = 0.0 bytes_in: int = 0 bytes_out: int = 0 + latency_samples: List[float] = field(default_factory=list) + + @staticmethod + def _compute_percentile(sorted_data: List[float], p: float) -> float: + if not sorted_data: + return 0.0 + k = (len(sorted_data) - 1) * (p / 100.0) + f = int(k) + c = min(f + 1, len(sorted_data) - 1) + d = k - f + return sorted_data[f] + d * (sorted_data[c] - sorted_data[f]) def record(self, latency_ms: float, success: bool, bytes_in: int = 0, bytes_out: int = 0) -> None: self.count += 1 @@ -36,10 +50,17 @@ class OperationStats: self.latency_max_ms = latency_ms self.bytes_in += bytes_in self.bytes_out += bytes_out + if len(self.latency_samples) < MAX_LATENCY_SAMPLES: + self.latency_samples.append(latency_ms) + else: + j = random.randint(0, self.count - 1) + if j < MAX_LATENCY_SAMPLES: + self.latency_samples[j] = latency_ms def to_dict(self) -> Dict[str, Any]: avg_latency = self.latency_sum_ms / self.count if self.count > 0 else 0.0 min_latency = self.latency_min_ms if self.latency_min_ms != float("inf") else 0.0 + sorted_latencies = sorted(self.latency_samples) return { "count": self.count, "success_count": self.success_count, @@ -47,6 +68,9 @@ class OperationStats: "latency_avg_ms": round(avg_latency, 2), "latency_min_ms": round(min_latency, 2), "latency_max_ms": round(self.latency_max_ms, 2), + "latency_p50_ms": round(self._compute_percentile(sorted_latencies, 50), 2), + "latency_p95_ms": round(self._compute_percentile(sorted_latencies, 95), 2), + "latency_p99_ms": round(self._compute_percentile(sorted_latencies, 99), 2), "bytes_in": self.bytes_in, "bytes_out": self.bytes_out, } @@ -62,6 +86,11 @@ class OperationStats: self.latency_max_ms = other.latency_max_ms self.bytes_in += other.bytes_in self.bytes_out += other.bytes_out + combined = self.latency_samples + other.latency_samples + if len(combined) > MAX_LATENCY_SAMPLES: + random.shuffle(combined) + combined = combined[:MAX_LATENCY_SAMPLES] + self.latency_samples = combined @dataclass diff --git a/app/s3_api.py b/app/s3_api.py index 8f3f7eb..7222ffa 100644 --- a/app/s3_api.py +++ b/app/s3_api.py @@ -17,6 +17,13 @@ from urllib.parse import quote, urlencode, urlparse, unquote from xml.etree.ElementTree import Element, SubElement, tostring, ParseError from defusedxml.ElementTree import fromstring +try: + import myfsio_core as _rc + _HAS_RUST = True +except ImportError: + _rc = None + _HAS_RUST = False + from flask import Blueprint, Response, current_app, jsonify, request, g from werkzeug.http import http_date @@ -192,11 +199,16 @@ _SIGNING_KEY_CACHE_MAX_SIZE = 256 def clear_signing_key_cache() -> None: + if _HAS_RUST: + _rc.clear_signing_key_cache() with _SIGNING_KEY_CACHE_LOCK: _SIGNING_KEY_CACHE.clear() def _get_signature_key(key: str, date_stamp: str, region_name: str, service_name: str) -> bytes: + if _HAS_RUST: + return bytes(_rc.derive_signing_key(key, date_stamp, region_name, service_name)) + cache_key = (key, date_stamp, region_name, service_name) now = time.time() @@ -255,39 +267,6 @@ def _verify_sigv4_header(req: Any, auth_header: str) -> Principal | None: if not secret_key: raise IamError("SignatureDoesNotMatch") - method = req.method - canonical_uri = _get_canonical_uri(req) - - query_args = [] - for key, value in req.args.items(multi=True): - query_args.append((key, value)) - query_args.sort(key=lambda x: (x[0], x[1])) - - canonical_query_parts = [] - for k, v in query_args: - canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}") - canonical_query_string = "&".join(canonical_query_parts) - - signed_headers_list = signed_headers_str.split(";") - canonical_headers_parts = [] - for header in signed_headers_list: - header_val = req.headers.get(header) - if header_val is None: - header_val = "" - - if header.lower() == 'expect' and header_val == "": - header_val = "100-continue" - - header_val = " ".join(header_val.split()) - canonical_headers_parts.append(f"{header.lower()}:{header_val}\n") - canonical_headers = "".join(canonical_headers_parts) - - payload_hash = req.headers.get("X-Amz-Content-Sha256") - if not payload_hash: - payload_hash = hashlib.sha256(req.get_data()).hexdigest() - - canonical_request = f"{method}\n{canonical_uri}\n{canonical_query_string}\n{canonical_headers}\n{signed_headers_str}\n{payload_hash}" - amz_date = req.headers.get("X-Amz-Date") or req.headers.get("Date") if not amz_date: raise IamError("Missing Date header") @@ -309,19 +288,60 @@ def _verify_sigv4_header(req: Any, auth_header: str) -> Principal | None: if 'date' in signed_headers_set: required_headers.remove('x-amz-date') required_headers.add('date') - + if not required_headers.issubset(signed_headers_set): raise IamError("Required headers not signed") - credential_scope = f"{date_stamp}/{region}/{service}/aws4_request" - string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}" - signing_key = _get_signature_key(secret_key, date_stamp, region, service) - calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() + canonical_uri = _get_canonical_uri(req) + payload_hash = req.headers.get("X-Amz-Content-Sha256") + if not payload_hash: + payload_hash = hashlib.sha256(req.get_data()).hexdigest() - if not hmac.compare_digest(calculated_signature, signature): - if current_app.config.get("DEBUG_SIGV4"): - logger.warning("SigV4 signature mismatch for %s %s", method, req.path) - raise IamError("SignatureDoesNotMatch") + if _HAS_RUST: + query_params = list(req.args.items(multi=True)) + header_values = [(h, req.headers.get(h) or "") for h in signed_headers_str.split(";")] + if not _rc.verify_sigv4_signature( + req.method, canonical_uri, query_params, signed_headers_str, + header_values, payload_hash, amz_date, date_stamp, region, + service, secret_key, signature, + ): + if current_app.config.get("DEBUG_SIGV4"): + logger.warning("SigV4 signature mismatch for %s %s", req.method, req.path) + raise IamError("SignatureDoesNotMatch") + else: + method = req.method + query_args = [] + for key, value in req.args.items(multi=True): + query_args.append((key, value)) + query_args.sort(key=lambda x: (x[0], x[1])) + + canonical_query_parts = [] + for k, v in query_args: + canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}") + canonical_query_string = "&".join(canonical_query_parts) + + signed_headers_list = signed_headers_str.split(";") + canonical_headers_parts = [] + for header in signed_headers_list: + header_val = req.headers.get(header) + if header_val is None: + header_val = "" + if header.lower() == 'expect' and header_val == "": + header_val = "100-continue" + header_val = " ".join(header_val.split()) + canonical_headers_parts.append(f"{header.lower()}:{header_val}\n") + canonical_headers = "".join(canonical_headers_parts) + + canonical_request = f"{method}\n{canonical_uri}\n{canonical_query_string}\n{canonical_headers}\n{signed_headers_str}\n{payload_hash}" + + credential_scope = f"{date_stamp}/{region}/{service}/aws4_request" + signing_key = _get_signature_key(secret_key, date_stamp, region, service) + string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}" + calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() + if not hmac.compare_digest(calculated_signature, signature): + if current_app.config.get("DEBUG_SIGV4"): + logger.warning("SigV4 signature mismatch for %s %s", method, req.path) + raise IamError("SignatureDoesNotMatch") session_token = req.headers.get("X-Amz-Security-Token") if session_token: @@ -350,14 +370,21 @@ def _verify_sigv4_query(req: Any) -> Principal | None: req_time = datetime.strptime(amz_date, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc) except ValueError: raise IamError("Invalid Date format") - + now = datetime.now(timezone.utc) + tolerance = timedelta(seconds=current_app.config.get("SIGV4_TIMESTAMP_TOLERANCE_SECONDS", 900)) + if req_time > now + tolerance: + raise IamError("Request date is too far in the future") try: expires_seconds = int(expires) if expires_seconds <= 0: raise IamError("Invalid Expires value: must be positive") except ValueError: raise IamError("Invalid Expires value: must be an integer") + min_expiry = current_app.config.get("PRESIGNED_URL_MIN_EXPIRY_SECONDS", 1) + max_expiry = current_app.config.get("PRESIGNED_URL_MAX_EXPIRY_SECONDS", 604800) + if expires_seconds < min_expiry or expires_seconds > max_expiry: + raise IamError(f"Expiration must be between {min_expiry} second(s) and {max_expiry} seconds") if now > req_time + timedelta(seconds=expires_seconds): raise IamError("Request expired") @@ -365,56 +392,58 @@ def _verify_sigv4_query(req: Any) -> Principal | None: if not secret_key: raise IamError("Invalid access key") - method = req.method canonical_uri = _get_canonical_uri(req) - - query_args = [] - for key, value in req.args.items(multi=True): - if key != "X-Amz-Signature": - query_args.append((key, value)) - query_args.sort(key=lambda x: (x[0], x[1])) - - canonical_query_parts = [] - for k, v in query_args: - canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}") - canonical_query_string = "&".join(canonical_query_parts) - - signed_headers_list = signed_headers_str.split(";") - canonical_headers_parts = [] - for header in signed_headers_list: - val = req.headers.get(header, "").strip() - if header.lower() == 'expect' and val == "": - val = "100-continue" - val = " ".join(val.split()) - canonical_headers_parts.append(f"{header.lower()}:{val}\n") - canonical_headers = "".join(canonical_headers_parts) - - payload_hash = "UNSIGNED-PAYLOAD" - - canonical_request = "\n".join([ - method, - canonical_uri, - canonical_query_string, - canonical_headers, - signed_headers_str, - payload_hash - ]) - - algorithm = "AWS4-HMAC-SHA256" - credential_scope = f"{date_stamp}/{region}/{service}/aws4_request" - hashed_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest() - string_to_sign = "\n".join([ - algorithm, - amz_date, - credential_scope, - hashed_request - ]) - - signing_key = _get_signature_key(secret_key, date_stamp, region, service) - calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() - if not hmac.compare_digest(calculated_signature, signature): - raise IamError("SignatureDoesNotMatch") + if _HAS_RUST: + query_params = [(k, v) for k, v in req.args.items(multi=True) if k != "X-Amz-Signature"] + header_values = [(h, req.headers.get(h) or "") for h in signed_headers_str.split(";")] + if not _rc.verify_sigv4_signature( + req.method, canonical_uri, query_params, signed_headers_str, + header_values, "UNSIGNED-PAYLOAD", amz_date, date_stamp, region, + service, secret_key, signature, + ): + raise IamError("SignatureDoesNotMatch") + else: + method = req.method + query_args = [] + for key, value in req.args.items(multi=True): + if key != "X-Amz-Signature": + query_args.append((key, value)) + query_args.sort(key=lambda x: (x[0], x[1])) + + canonical_query_parts = [] + for k, v in query_args: + canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}") + canonical_query_string = "&".join(canonical_query_parts) + + signed_headers_list = signed_headers_str.split(";") + canonical_headers_parts = [] + for header in signed_headers_list: + val = req.headers.get(header, "").strip() + if header.lower() == 'expect' and val == "": + val = "100-continue" + val = " ".join(val.split()) + canonical_headers_parts.append(f"{header.lower()}:{val}\n") + canonical_headers = "".join(canonical_headers_parts) + + payload_hash = "UNSIGNED-PAYLOAD" + + canonical_request = "\n".join([ + method, + canonical_uri, + canonical_query_string, + canonical_headers, + signed_headers_str, + payload_hash + ]) + + credential_scope = f"{date_stamp}/{region}/{service}/aws4_request" + signing_key = _get_signature_key(secret_key, date_stamp, region, service) + hashed_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest() + string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashed_request}" + calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() + if not hmac.compare_digest(calculated_signature, signature): + raise IamError("SignatureDoesNotMatch") session_token = req.args.get("X-Amz-Security-Token") if session_token: @@ -573,7 +602,11 @@ def _validate_presigned_request(action: str, bucket_name: str, object_key: str) request_time = datetime.strptime(amz_date, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc) except ValueError as exc: raise IamError("Invalid X-Amz-Date") from exc - if datetime.now(timezone.utc) > request_time + timedelta(seconds=expiry): + now = datetime.now(timezone.utc) + tolerance = timedelta(seconds=current_app.config.get("SIGV4_TIMESTAMP_TOLERANCE_SECONDS", 900)) + if request_time > now + tolerance: + raise IamError("Request date is too far in the future") + if now > request_time + timedelta(seconds=expiry): raise IamError("Presigned URL expired") signed_headers_list = [header.strip().lower() for header in signed_headers.split(";") if header] @@ -973,7 +1006,7 @@ def _render_encryption_document(config: dict[str, Any]) -> Element: return root -def _stream_file(path, chunk_size: int = 64 * 1024): +def _stream_file(path, chunk_size: int = 256 * 1024): with path.open("rb") as handle: while True: chunk = handle.read(chunk_size) @@ -1026,6 +1059,7 @@ def _maybe_handle_bucket_subresource(bucket_name: str) -> Response | None: "logging": _bucket_logging_handler, "uploads": _bucket_uploads_handler, "policy": _bucket_policy_handler, + "policyStatus": _bucket_policy_status_handler, "replication": _bucket_replication_handler, "website": _bucket_website_handler, } @@ -1308,8 +1342,8 @@ def _bucket_cors_handler(bucket_name: str) -> Response: def _bucket_encryption_handler(bucket_name: str) -> Response: - if request.method not in {"GET", "PUT"}: - return _method_not_allowed(["GET", "PUT"]) + if request.method not in {"GET", "PUT", "DELETE"}: + return _method_not_allowed(["GET", "PUT", "DELETE"]) principal, error = _require_principal() if error: return error @@ -1330,6 +1364,13 @@ def _bucket_encryption_handler(bucket_name: str) -> Response: 404, ) return _xml_response(_render_encryption_document(config)) + if request.method == "DELETE": + try: + storage.set_bucket_encryption(bucket_name, None) + except StorageError as exc: + return _error_response("NoSuchBucket", str(exc), 404) + current_app.logger.info("Bucket encryption deleted", extra={"bucket": bucket_name}) + return Response(status=204) ct_error = _require_xml_content_type() if ct_error: return ct_error @@ -1426,6 +1467,99 @@ def _bucket_acl_handler(bucket_name: str) -> Response: return _xml_response(root) +def _object_acl_handler(bucket_name: str, object_key: str) -> Response: + from .acl import create_canned_acl, GRANTEE_ALL_USERS, GRANTEE_AUTHENTICATED_USERS + + if request.method not in {"GET", "PUT"}: + return _method_not_allowed(["GET", "PUT"]) + storage = _storage() + try: + path = storage.get_object_path(bucket_name, object_key) + except (StorageError, FileNotFoundError): + return _error_response("NoSuchKey", "Object not found", 404) + + if request.method == "PUT": + principal, error = _object_principal("write", bucket_name, object_key) + if error: + return error + owner_id = principal.access_key if principal else "anonymous" + canned_acl = request.headers.get("x-amz-acl", "private") + acl = create_canned_acl(canned_acl, owner_id) + acl_service = _acl() + metadata = storage.get_object_metadata(bucket_name, object_key) + metadata.update(acl_service.create_object_acl_metadata(acl)) + safe_key = storage._sanitize_object_key(object_key, storage._object_key_max_length_bytes) + storage._write_metadata(bucket_name, safe_key, metadata) + current_app.logger.info("Object ACL set", extra={"bucket": bucket_name, "key": object_key, "acl": canned_acl}) + return Response(status=200) + + principal, error = _object_principal("read", bucket_name, object_key) + if error: + return error + owner_id = principal.access_key if principal else "anonymous" + acl_service = _acl() + metadata = storage.get_object_metadata(bucket_name, object_key) + acl = acl_service.get_object_acl(bucket_name, object_key, metadata) + if not acl: + acl = create_canned_acl("private", owner_id) + + root = Element("AccessControlPolicy") + owner_el = SubElement(root, "Owner") + SubElement(owner_el, "ID").text = acl.owner + SubElement(owner_el, "DisplayName").text = acl.owner + acl_el = SubElement(root, "AccessControlList") + for grant in acl.grants: + grant_el = SubElement(acl_el, "Grant") + grantee = SubElement(grant_el, "Grantee") + if grant.grantee == GRANTEE_ALL_USERS: + grantee.set("{http://www.w3.org/2001/XMLSchema-instance}type", "Group") + SubElement(grantee, "URI").text = "http://acs.amazonaws.com/groups/global/AllUsers" + elif grant.grantee == GRANTEE_AUTHENTICATED_USERS: + grantee.set("{http://www.w3.org/2001/XMLSchema-instance}type", "Group") + SubElement(grantee, "URI").text = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" + else: + grantee.set("{http://www.w3.org/2001/XMLSchema-instance}type", "CanonicalUser") + SubElement(grantee, "ID").text = grant.grantee + SubElement(grantee, "DisplayName").text = grant.grantee + SubElement(grant_el, "Permission").text = grant.permission + return _xml_response(root) + + +def _object_attributes_handler(bucket_name: str, object_key: str) -> Response: + if request.method != "GET": + return _method_not_allowed(["GET"]) + principal, error = _object_principal("read", bucket_name, object_key) + if error: + return error + storage = _storage() + try: + path = storage.get_object_path(bucket_name, object_key) + file_stat = path.stat() + metadata = storage.get_object_metadata(bucket_name, object_key) + except (StorageError, FileNotFoundError): + return _error_response("NoSuchKey", "Object not found", 404) + + requested = request.headers.get("x-amz-object-attributes", "") + attrs = {a.strip() for a in requested.split(",") if a.strip()} + + root = Element("GetObjectAttributesResponse") + if "ETag" in attrs: + etag = metadata.get("__etag__") or storage._compute_etag(path) + SubElement(root, "ETag").text = etag + if "StorageClass" in attrs: + SubElement(root, "StorageClass").text = "STANDARD" + if "ObjectSize" in attrs: + SubElement(root, "ObjectSize").text = str(file_stat.st_size) + if "Checksum" in attrs: + SubElement(root, "Checksum") + if "ObjectParts" in attrs: + SubElement(root, "ObjectParts") + + response = _xml_response(root) + response.headers["Last-Modified"] = http_date(file_stat.st_mtime) + return response + + def _bucket_list_versions_handler(bucket_name: str) -> Response: """Handle ListObjectVersions (GET /?versions).""" if request.method != "GET": @@ -2347,6 +2481,10 @@ def _post_object(bucket_name: str) -> Response: if success_action_redirect: allowed_hosts = current_app.config.get("ALLOWED_REDIRECT_HOSTS", []) if not allowed_hosts: + current_app.logger.warning( + "ALLOWED_REDIRECT_HOSTS not configured, falling back to request Host header. " + "Set ALLOWED_REDIRECT_HOSTS for production deployments." + ) allowed_hosts = [request.host] parsed = urlparse(success_action_redirect) if parsed.scheme not in ("http", "https"): @@ -2656,6 +2794,12 @@ def object_handler(bucket_name: str, object_key: str): if "legal-hold" in request.args: return _object_legal_hold_handler(bucket_name, object_key) + if "acl" in request.args: + return _object_acl_handler(bucket_name, object_key) + + if "attributes" in request.args: + return _object_attributes_handler(bucket_name, object_key) + if request.method == "POST": if "uploads" in request.args: return _initiate_multipart_upload(bucket_name, object_key) @@ -2803,7 +2947,7 @@ def object_handler(bucket_name: str, object_key: str): f.seek(start_pos) remaining = length_to_read while remaining > 0: - chunk_size = min(65536, remaining) + chunk_size = min(262144, remaining) chunk = f.read(chunk_size) if not chunk: break @@ -2980,6 +3124,32 @@ def _bucket_policy_handler(bucket_name: str) -> Response: return Response(status=204) +def _bucket_policy_status_handler(bucket_name: str) -> Response: + if request.method != "GET": + return _method_not_allowed(["GET"]) + principal, error = _require_principal() + if error: + return error + try: + _authorize_action(principal, bucket_name, "policy") + except IamError as exc: + return _error_response("AccessDenied", str(exc), 403) + storage = _storage() + if not storage.bucket_exists(bucket_name): + return _error_response("NoSuchBucket", "Bucket does not exist", 404) + store = _bucket_policies() + policy = store.get_policy(bucket_name) + is_public = False + if policy: + for statement in policy.get("Statement", []): + if statement.get("Effect") == "Allow" and statement.get("Principal") == "*": + is_public = True + break + root = Element("PolicyStatus") + SubElement(root, "IsPublic").text = "TRUE" if is_public else "FALSE" + return _xml_response(root) + + def _bucket_replication_handler(bucket_name: str) -> Response: if request.method not in {"GET", "PUT", "DELETE"}: return _method_not_allowed(["GET", "PUT", "DELETE"]) @@ -3193,7 +3363,7 @@ def head_object(bucket_name: str, object_key: str) -> Response: path = _storage().get_object_path(bucket_name, object_key) metadata = _storage().get_object_metadata(bucket_name, object_key) stat = path.stat() - etag = _storage()._compute_etag(path) + etag = metadata.get("__etag__") or _storage()._compute_etag(path) response = Response(status=200) _apply_object_headers(response, file_stat=stat, metadata=metadata, etag=etag) diff --git a/app/storage.py b/app/storage.py index 3ef3d0d..683fd4b 100644 --- a/app/storage.py +++ b/app/storage.py @@ -1,5 +1,6 @@ from __future__ import annotations +import copy import hashlib import json import os @@ -18,6 +19,13 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any, BinaryIO, Dict, Generator, List, Optional +try: + import myfsio_core as _rc + _HAS_RUST = True +except ImportError: + _rc = None + _HAS_RUST = False + # Platform-specific file locking if os.name == "nt": import msvcrt @@ -189,6 +197,8 @@ class ObjectStorage: self._object_key_max_length_bytes = object_key_max_length_bytes self._sorted_key_cache: Dict[str, tuple[list[str], int]] = {} self._meta_index_locks: Dict[str, threading.Lock] = {} + self._meta_read_cache: OrderedDict[tuple, Optional[Dict[str, Any]]] = OrderedDict() + self._meta_read_cache_max = 2048 self._cleanup_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="ParentCleanup") def _get_bucket_lock(self, bucket_id: str) -> threading.Lock: @@ -220,6 +230,11 @@ class ObjectStorage: raise BucketNotFoundError("Bucket does not exist") def _validate_bucket_name(self, bucket_name: str) -> None: + if _HAS_RUST: + error = _rc.validate_bucket_name(bucket_name) + if error: + raise StorageError(error) + return if len(bucket_name) < 3 or len(bucket_name) > 63: raise StorageError("Bucket name must be between 3 and 63 characters") if not re.match(r"^[a-z0-9][a-z0-9.-]*[a-z0-9]$", bucket_name): @@ -1892,14 +1907,38 @@ class ObjectStorage: return self._meta_index_locks[index_path] def _read_index_entry(self, bucket_name: str, key: Path) -> Optional[Dict[str, Any]]: + cache_key = (bucket_name, str(key)) + with self._cache_lock: + hit = self._meta_read_cache.get(cache_key) + if hit is not None: + self._meta_read_cache.move_to_end(cache_key) + cached = hit[0] + return copy.deepcopy(cached) if cached is not None else None + index_path, entry_name = self._index_file_for_key(bucket_name, key) - if not index_path.exists(): - return None - try: - index_data = json.loads(index_path.read_text(encoding="utf-8")) - return index_data.get(entry_name) - except (OSError, json.JSONDecodeError): - return None + if _HAS_RUST: + result = _rc.read_index_entry(str(index_path), entry_name) + else: + if not index_path.exists(): + result = None + else: + try: + index_data = json.loads(index_path.read_text(encoding="utf-8")) + result = index_data.get(entry_name) + except (OSError, json.JSONDecodeError): + result = None + + with self._cache_lock: + while len(self._meta_read_cache) >= self._meta_read_cache_max: + self._meta_read_cache.popitem(last=False) + self._meta_read_cache[cache_key] = (copy.deepcopy(result) if result is not None else None,) + + return result + + def _invalidate_meta_read_cache(self, bucket_name: str, key: Path) -> None: + cache_key = (bucket_name, str(key)) + with self._cache_lock: + self._meta_read_cache.pop(cache_key, None) def _write_index_entry(self, bucket_name: str, key: Path, entry: Dict[str, Any]) -> None: index_path, entry_name = self._index_file_for_key(bucket_name, key) @@ -1914,16 +1953,19 @@ class ObjectStorage: pass index_data[entry_name] = entry index_path.write_text(json.dumps(index_data), encoding="utf-8") + self._invalidate_meta_read_cache(bucket_name, key) def _delete_index_entry(self, bucket_name: str, key: Path) -> None: index_path, entry_name = self._index_file_for_key(bucket_name, key) if not index_path.exists(): + self._invalidate_meta_read_cache(bucket_name, key) return lock = self._get_meta_index_lock(str(index_path)) with lock: try: index_data = json.loads(index_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): + self._invalidate_meta_read_cache(bucket_name, key) return if entry_name in index_data: del index_data[entry_name] @@ -1934,6 +1976,7 @@ class ObjectStorage: index_path.unlink() except OSError: pass + self._invalidate_meta_read_cache(bucket_name, key) def _normalize_metadata(self, metadata: Optional[Dict[str, str]]) -> Optional[Dict[str, str]]: if not metadata: @@ -2133,6 +2176,18 @@ class ObjectStorage: @staticmethod def _sanitize_object_key(object_key: str, max_length_bytes: int = 1024) -> Path: + if _HAS_RUST: + error = _rc.validate_object_key(object_key, max_length_bytes, os.name == "nt") + if error: + raise StorageError(error) + normalized = unicodedata.normalize("NFC", object_key) + candidate = Path(normalized) + if candidate.is_absolute(): + raise StorageError("Absolute object keys are not allowed") + if getattr(candidate, "drive", ""): + raise StorageError("Object key cannot include a drive letter") + return Path(*candidate.parts) if candidate.parts else candidate + if not object_key: raise StorageError("Object key required") if "\x00" in object_key: @@ -2146,7 +2201,7 @@ class ObjectStorage: candidate = Path(object_key) if ".." in candidate.parts: raise StorageError("Object key contains parent directory references") - + if candidate.is_absolute(): raise StorageError("Absolute object keys are not allowed") if getattr(candidate, "drive", ""): @@ -2174,6 +2229,8 @@ class ObjectStorage: @staticmethod def _compute_etag(path: Path) -> str: + if _HAS_RUST: + return _rc.md5_file(str(path)) checksum = hashlib.md5() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(8192), b""): diff --git a/app/ui.py b/app/ui.py index 9124bbb..7dfaf90 100644 --- a/app/ui.py +++ b/app/ui.py @@ -51,6 +51,7 @@ from .s3_client import ( from .secret_store import EphemeralSecretStore from .site_registry import SiteRegistry, SiteInfo, PeerSite from .storage import ObjectStorage, StorageError +from .website_domains import normalize_domain, is_valid_domain ui_bp = Blueprint("ui", __name__, template_folder="../templates", url_prefix="/ui") @@ -507,11 +508,15 @@ def bucket_detail(bucket_name: str): can_manage_quota = is_replication_admin website_config = None + website_domains = [] if website_hosting_enabled: try: website_config = storage.get_bucket_website(bucket_name) except StorageError: website_config = None + domain_store = current_app.extensions.get("website_domains") + if domain_store: + website_domains = domain_store.get_domains_for_bucket(bucket_name) objects_api_url = url_for("ui.list_bucket_objects", bucket_name=bucket_name) objects_stream_url = url_for("ui.stream_bucket_objects", bucket_name=bucket_name) @@ -557,6 +562,7 @@ def bucket_detail(bucket_name: str): site_sync_enabled=site_sync_enabled, website_hosting_enabled=website_hosting_enabled, website_config=website_config, + website_domains=website_domains, can_manage_website=can_edit_policy, ) @@ -737,7 +743,6 @@ def initiate_multipart_upload(bucket_name: str): @ui_bp.put("/buckets//multipart//parts") -@limiter.exempt @csrf.exempt def upload_multipart_part(bucket_name: str, upload_id: str): principal = _current_principal() @@ -2042,16 +2047,17 @@ def update_connection(connection_id: str): secret_key = request.form.get("secret_key", "").strip() region = request.form.get("region", "us-east-1").strip() - if not all([name, endpoint, access_key, secret_key]): + if not all([name, endpoint, access_key]): if _wants_json(): - return jsonify({"error": "All fields are required"}), 400 - flash("All fields are required", "danger") + return jsonify({"error": "Name, endpoint, and access key are required"}), 400 + flash("Name, endpoint, and access key are required", "danger") return redirect(url_for("ui.connections_dashboard")) conn.name = name conn.endpoint_url = endpoint conn.access_key = access_key - conn.secret_key = secret_key + if secret_key: + conn.secret_key = secret_key conn.region = region _connections().save() @@ -2372,7 +2378,10 @@ def website_domains_dashboard(): store = current_app.extensions.get("website_domains") mappings = store.list_all() if store else [] storage = _storage() - buckets = [b.name for b in storage.list_buckets()] + buckets = [ + b.name for b in storage.list_buckets() + if storage.get_bucket_website(b.name) + ] return render_template( "website_domains.html", mappings=mappings, @@ -2399,7 +2408,7 @@ def create_website_domain(): flash("Website hosting is not enabled", "warning") return redirect(url_for("ui.buckets_overview")) - domain = (request.form.get("domain") or "").strip().lower() + domain = normalize_domain(request.form.get("domain") or "") bucket = (request.form.get("bucket") or "").strip() if not domain: @@ -2408,6 +2417,12 @@ def create_website_domain(): flash("Domain is required", "danger") return redirect(url_for("ui.website_domains_dashboard")) + if not is_valid_domain(domain): + if _wants_json(): + return jsonify({"error": f"Invalid domain format: '{domain}'"}), 400 + flash(f"Invalid domain format: '{domain}'. Use a hostname like www.example.com", "danger") + return redirect(url_for("ui.website_domains_dashboard")) + if not bucket: if _wants_json(): return jsonify({"error": "Bucket is required"}), 400 @@ -2446,6 +2461,7 @@ def update_website_domain(domain: str): flash("Access denied", "danger") return redirect(url_for("ui.website_domains_dashboard")) + domain = normalize_domain(domain) bucket = (request.form.get("bucket") or "").strip() if not bucket: if _wants_json(): @@ -2461,9 +2477,14 @@ def update_website_domain(domain: str): return redirect(url_for("ui.website_domains_dashboard")) store = current_app.extensions.get("website_domains") + if not store.get_bucket(domain): + if _wants_json(): + return jsonify({"error": f"No mapping for domain '{domain}'"}), 404 + flash(f"No mapping for domain '{domain}'", "danger") + return redirect(url_for("ui.website_domains_dashboard")) store.set_mapping(domain, bucket) if _wants_json(): - return jsonify({"success": True, "domain": domain.lower(), "bucket": bucket}) + return jsonify({"success": True, "domain": domain, "bucket": bucket}) flash(f"Domain '{domain}' updated to bucket '{bucket}'", "success") return redirect(url_for("ui.website_domains_dashboard")) @@ -2479,6 +2500,7 @@ def delete_website_domain(domain: str): flash("Access denied", "danger") return redirect(url_for("ui.website_domains_dashboard")) + domain = normalize_domain(domain) store = current_app.extensions.get("website_domains") if not store.delete_mapping(domain): if _wants_json(): @@ -3278,9 +3300,12 @@ def sites_dashboard(): @ui_bp.post("/sites/local") def update_local_site(): principal = _current_principal() + wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest" try: _iam().authorize(principal, None, "iam:*") except IamError: + if wants_json: + return jsonify({"error": "Access denied"}), 403 flash("Access denied", "danger") return redirect(url_for("ui.sites_dashboard")) @@ -3291,6 +3316,8 @@ def update_local_site(): display_name = request.form.get("display_name", "").strip() if not site_id: + if wants_json: + return jsonify({"error": "Site ID is required"}), 400 flash("Site ID is required", "danger") return redirect(url_for("ui.sites_dashboard")) @@ -3312,6 +3339,8 @@ def update_local_site(): ) registry.set_local_site(site) + if wants_json: + return jsonify({"message": "Local site configuration updated"}) flash("Local site configuration updated", "success") return redirect(url_for("ui.sites_dashboard")) @@ -3319,9 +3348,12 @@ def update_local_site(): @ui_bp.post("/sites/peers") def add_peer_site(): principal = _current_principal() + wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest" try: _iam().authorize(principal, None, "iam:*") except IamError: + if wants_json: + return jsonify({"error": "Access denied"}), 403 flash("Access denied", "danger") return redirect(url_for("ui.sites_dashboard")) @@ -3333,9 +3365,13 @@ def add_peer_site(): connection_id = request.form.get("connection_id", "").strip() or None if not site_id: + if wants_json: + return jsonify({"error": "Site ID is required"}), 400 flash("Site ID is required", "danger") return redirect(url_for("ui.sites_dashboard")) if not endpoint: + if wants_json: + return jsonify({"error": "Endpoint is required"}), 400 flash("Endpoint is required", "danger") return redirect(url_for("ui.sites_dashboard")) @@ -3347,10 +3383,14 @@ def add_peer_site(): registry = _site_registry() if registry.get_peer(site_id): + if wants_json: + return jsonify({"error": f"Peer site '{site_id}' already exists"}), 409 flash(f"Peer site '{site_id}' already exists", "danger") return redirect(url_for("ui.sites_dashboard")) if connection_id and not _connections().get(connection_id): + if wants_json: + return jsonify({"error": f"Connection '{connection_id}' not found"}), 404 flash(f"Connection '{connection_id}' not found", "danger") return redirect(url_for("ui.sites_dashboard")) @@ -3364,6 +3404,11 @@ def add_peer_site(): ) registry.add_peer(peer) + if wants_json: + redirect_url = None + if connection_id: + redirect_url = url_for("ui.replication_wizard", site_id=site_id) + return jsonify({"message": f"Peer site '{site_id}' added", "redirect": redirect_url}) flash(f"Peer site '{site_id}' added", "success") if connection_id: @@ -3374,9 +3419,12 @@ def add_peer_site(): @ui_bp.post("/sites/peers//update") def update_peer_site(site_id: str): principal = _current_principal() + wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest" try: _iam().authorize(principal, None, "iam:*") except IamError: + if wants_json: + return jsonify({"error": "Access denied"}), 403 flash("Access denied", "danger") return redirect(url_for("ui.sites_dashboard")) @@ -3384,6 +3432,8 @@ def update_peer_site(site_id: str): existing = registry.get_peer(site_id) if not existing: + if wants_json: + return jsonify({"error": f"Peer site '{site_id}' not found"}), 404 flash(f"Peer site '{site_id}' not found", "danger") return redirect(url_for("ui.sites_dashboard")) @@ -3391,7 +3441,10 @@ def update_peer_site(site_id: str): region = request.form.get("region", existing.region).strip() priority = request.form.get("priority", str(existing.priority)) display_name = request.form.get("display_name", existing.display_name).strip() - connection_id = request.form.get("connection_id", "").strip() or existing.connection_id + if "connection_id" in request.form: + connection_id = request.form["connection_id"].strip() or None + else: + connection_id = existing.connection_id try: priority_int = int(priority) @@ -3399,6 +3452,8 @@ def update_peer_site(site_id: str): priority_int = existing.priority if connection_id and not _connections().get(connection_id): + if wants_json: + return jsonify({"error": f"Connection '{connection_id}' not found"}), 404 flash(f"Connection '{connection_id}' not found", "danger") return redirect(url_for("ui.sites_dashboard")) @@ -3415,6 +3470,8 @@ def update_peer_site(site_id: str): ) registry.update_peer(peer) + if wants_json: + return jsonify({"message": f"Peer site '{site_id}' updated"}) flash(f"Peer site '{site_id}' updated", "success") return redirect(url_for("ui.sites_dashboard")) @@ -3422,16 +3479,23 @@ def update_peer_site(site_id: str): @ui_bp.post("/sites/peers//delete") def delete_peer_site(site_id: str): principal = _current_principal() + wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest" try: _iam().authorize(principal, None, "iam:*") except IamError: + if wants_json: + return jsonify({"error": "Access denied"}), 403 flash("Access denied", "danger") return redirect(url_for("ui.sites_dashboard")) registry = _site_registry() if registry.delete_peer(site_id): + if wants_json: + return jsonify({"message": f"Peer site '{site_id}' deleted"}) flash(f"Peer site '{site_id}' deleted", "success") else: + if wants_json: + return jsonify({"error": f"Peer site '{site_id}' not found"}), 404 flash(f"Peer site '{site_id}' not found", "danger") return redirect(url_for("ui.sites_dashboard")) diff --git a/app/version.py b/app/version.py index b25ea84..fc8981e 100644 --- a/app/version.py +++ b/app/version.py @@ -1,6 +1,6 @@ from __future__ import annotations -APP_VERSION = "0.2.9" +APP_VERSION = "0.3.0" def get_version() -> str: diff --git a/app/website_domains.py b/app/website_domains.py index 4dc4044..7ec33f7 100644 --- a/app/website_domains.py +++ b/app/website_domains.py @@ -1,23 +1,50 @@ from __future__ import annotations import json +import re import threading from pathlib import Path from typing import Dict, List, Optional +_DOMAIN_RE = re.compile( + r"^(?!-)[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$" +) + + +def normalize_domain(raw: str) -> str: + raw = raw.strip().lower() + for prefix in ("https://", "http://"): + if raw.startswith(prefix): + raw = raw[len(prefix):] + raw = raw.split("/", 1)[0] + raw = raw.split("?", 1)[0] + raw = raw.split("#", 1)[0] + if ":" in raw: + raw = raw.rsplit(":", 1)[0] + return raw + + +def is_valid_domain(domain: str) -> bool: + if not domain or len(domain) > 253: + return False + return bool(_DOMAIN_RE.match(domain)) + class WebsiteDomainStore: def __init__(self, config_path: Path) -> None: self.config_path = config_path self._lock = threading.Lock() self._domains: Dict[str, str] = {} + self._last_mtime: float = 0.0 self.reload() def reload(self) -> None: if not self.config_path.exists(): self._domains = {} + self._last_mtime = 0.0 return try: + self._last_mtime = self.config_path.stat().st_mtime with open(self.config_path, "r", encoding="utf-8") as f: data = json.load(f) if isinstance(data, dict): @@ -27,19 +54,45 @@ class WebsiteDomainStore: except (OSError, json.JSONDecodeError): self._domains = {} + def _maybe_reload(self) -> None: + try: + if self.config_path.exists(): + mtime = self.config_path.stat().st_mtime + if mtime != self._last_mtime: + self._last_mtime = mtime + with open(self.config_path, "r", encoding="utf-8") as f: + data = json.load(f) + if isinstance(data, dict): + self._domains = {k.lower(): v for k, v in data.items()} + else: + self._domains = {} + elif self._domains: + self._domains = {} + self._last_mtime = 0.0 + except (OSError, json.JSONDecodeError): + pass + def _save(self) -> None: self.config_path.parent.mkdir(parents=True, exist_ok=True) with open(self.config_path, "w", encoding="utf-8") as f: json.dump(self._domains, f, indent=2) + self._last_mtime = self.config_path.stat().st_mtime def list_all(self) -> List[Dict[str, str]]: with self._lock: + self._maybe_reload() return [{"domain": d, "bucket": b} for d, b in self._domains.items()] def get_bucket(self, domain: str) -> Optional[str]: with self._lock: + self._maybe_reload() return self._domains.get(domain.lower()) + def get_domains_for_bucket(self, bucket: str) -> List[str]: + with self._lock: + self._maybe_reload() + return [d for d, b in self._domains.items() if b == bucket] + def set_mapping(self, domain: str, bucket: str) -> None: with self._lock: self._domains[domain.lower()] = bucket diff --git a/myfsio_core/Cargo.toml b/myfsio_core/Cargo.toml new file mode 100644 index 0000000..2bff9cc --- /dev/null +++ b/myfsio_core/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "myfsio_core" +version = "0.1.0" +edition = "2021" + +[lib] +name = "myfsio_core" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = { version = "0.28", features = ["extension-module"] } +hmac = "0.12" +sha2 = "0.10" +md-5 = "0.10" +hex = "0.4" +unicode-normalization = "0.1" +serde_json = "1" +regex = "1" +lru = "0.14" +parking_lot = "0.12" +percent-encoding = "2" diff --git a/myfsio_core/pyproject.toml b/myfsio_core/pyproject.toml new file mode 100644 index 0000000..fbea25c --- /dev/null +++ b/myfsio_core/pyproject.toml @@ -0,0 +1,11 @@ +[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[project] +name = "myfsio_core" +version = "0.1.0" +requires-python = ">=3.10" + +[tool.maturin] +features = ["pyo3/extension-module"] diff --git a/myfsio_core/src/hashing.rs b/myfsio_core/src/hashing.rs new file mode 100644 index 0000000..aec3fa4 --- /dev/null +++ b/myfsio_core/src/hashing.rs @@ -0,0 +1,90 @@ +use md5::{Digest, Md5}; +use pyo3::exceptions::PyIOError; +use pyo3::prelude::*; +use sha2::Sha256; +use std::fs::File; +use std::io::Read; + +const CHUNK_SIZE: usize = 65536; + +#[pyfunction] +pub fn md5_file(py: Python<'_>, path: &str) -> PyResult { + let path = path.to_owned(); + py.detach(move || { + let mut file = File::open(&path) + .map_err(|e| PyIOError::new_err(format!("Failed to open file: {}", e)))?; + let mut hasher = Md5::new(); + let mut buf = vec![0u8; CHUNK_SIZE]; + loop { + let n = file + .read(&mut buf) + .map_err(|e| PyIOError::new_err(format!("Failed to read file: {}", e)))?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + Ok(format!("{:x}", hasher.finalize())) + }) +} + +#[pyfunction] +pub fn md5_bytes(data: &[u8]) -> String { + let mut hasher = Md5::new(); + hasher.update(data); + format!("{:x}", hasher.finalize()) +} + +#[pyfunction] +pub fn sha256_file(py: Python<'_>, path: &str) -> PyResult { + let path = path.to_owned(); + py.detach(move || { + let mut file = File::open(&path) + .map_err(|e| PyIOError::new_err(format!("Failed to open file: {}", e)))?; + let mut hasher = Sha256::new(); + let mut buf = vec![0u8; CHUNK_SIZE]; + loop { + let n = file + .read(&mut buf) + .map_err(|e| PyIOError::new_err(format!("Failed to read file: {}", e)))?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + Ok(format!("{:x}", hasher.finalize())) + }) +} + +#[pyfunction] +pub fn sha256_bytes(data: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(data); + format!("{:x}", hasher.finalize()) +} + +#[pyfunction] +pub fn md5_sha256_file(py: Python<'_>, path: &str) -> PyResult<(String, String)> { + let path = path.to_owned(); + py.detach(move || { + let mut file = File::open(&path) + .map_err(|e| PyIOError::new_err(format!("Failed to open file: {}", e)))?; + let mut md5_hasher = Md5::new(); + let mut sha_hasher = Sha256::new(); + let mut buf = vec![0u8; CHUNK_SIZE]; + loop { + let n = file + .read(&mut buf) + .map_err(|e| PyIOError::new_err(format!("Failed to read file: {}", e)))?; + if n == 0 { + break; + } + md5_hasher.update(&buf[..n]); + sha_hasher.update(&buf[..n]); + } + Ok(( + format!("{:x}", md5_hasher.finalize()), + format!("{:x}", sha_hasher.finalize()), + )) + }) +} diff --git a/myfsio_core/src/lib.rs b/myfsio_core/src/lib.rs new file mode 100644 index 0000000..fc1b9f3 --- /dev/null +++ b/myfsio_core/src/lib.rs @@ -0,0 +1,34 @@ +mod hashing; +mod metadata; +mod sigv4; +mod validation; + +use pyo3::prelude::*; + +#[pymodule] +mod myfsio_core { + use super::*; + + #[pymodule_init] + fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(sigv4::verify_sigv4_signature, m)?)?; + m.add_function(wrap_pyfunction!(sigv4::derive_signing_key, m)?)?; + m.add_function(wrap_pyfunction!(sigv4::compute_signature, m)?)?; + m.add_function(wrap_pyfunction!(sigv4::build_string_to_sign, m)?)?; + m.add_function(wrap_pyfunction!(sigv4::constant_time_compare, m)?)?; + m.add_function(wrap_pyfunction!(sigv4::clear_signing_key_cache, m)?)?; + + m.add_function(wrap_pyfunction!(hashing::md5_file, m)?)?; + m.add_function(wrap_pyfunction!(hashing::md5_bytes, m)?)?; + m.add_function(wrap_pyfunction!(hashing::sha256_file, m)?)?; + m.add_function(wrap_pyfunction!(hashing::sha256_bytes, m)?)?; + m.add_function(wrap_pyfunction!(hashing::md5_sha256_file, m)?)?; + + m.add_function(wrap_pyfunction!(validation::validate_object_key, m)?)?; + m.add_function(wrap_pyfunction!(validation::validate_bucket_name, m)?)?; + + m.add_function(wrap_pyfunction!(metadata::read_index_entry, m)?)?; + + Ok(()) + } +} diff --git a/myfsio_core/src/metadata.rs b/myfsio_core/src/metadata.rs new file mode 100644 index 0000000..67d09f8 --- /dev/null +++ b/myfsio_core/src/metadata.rs @@ -0,0 +1,71 @@ +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyList, PyString}; +use serde_json::Value; +use std::fs; + +const MAX_DEPTH: u32 = 64; + +fn value_to_py(py: Python<'_>, v: &Value, depth: u32) -> PyResult> { + if depth > MAX_DEPTH { + return Err(PyValueError::new_err("JSON nesting too deep")); + } + match v { + Value::Null => Ok(py.None()), + Value::Bool(b) => Ok((*b).into_pyobject(py)?.to_owned().into_any().unbind()), + Value::Number(n) => { + if let Some(i) = n.as_i64() { + Ok(i.into_pyobject(py)?.into_any().unbind()) + } else if let Some(f) = n.as_f64() { + Ok(f.into_pyobject(py)?.into_any().unbind()) + } else { + Ok(py.None()) + } + } + Value::String(s) => Ok(PyString::new(py, s).into_any().unbind()), + Value::Array(arr) => { + let list = PyList::empty(py); + for item in arr { + list.append(value_to_py(py, item, depth + 1)?)?; + } + Ok(list.into_any().unbind()) + } + Value::Object(map) => { + let dict = PyDict::new(py); + for (k, val) in map { + dict.set_item(k, value_to_py(py, val, depth + 1)?)?; + } + Ok(dict.into_any().unbind()) + } + } +} + +#[pyfunction] +pub fn read_index_entry( + py: Python<'_>, + path: &str, + entry_name: &str, +) -> PyResult>> { + let path_owned = path.to_owned(); + let entry_owned = entry_name.to_owned(); + + let entry: Option = py.detach(move || -> PyResult> { + let content = match fs::read_to_string(&path_owned) { + Ok(c) => c, + Err(_) => return Ok(None), + }; + let parsed: Value = match serde_json::from_str(&content) { + Ok(v) => v, + Err(_) => return Ok(None), + }; + match parsed { + Value::Object(mut map) => Ok(map.remove(&entry_owned)), + _ => Ok(None), + } + })?; + + match entry { + Some(val) => Ok(Some(value_to_py(py, &val, 0)?)), + None => Ok(None), + } +} diff --git a/myfsio_core/src/sigv4.rs b/myfsio_core/src/sigv4.rs new file mode 100644 index 0000000..904a853 --- /dev/null +++ b/myfsio_core/src/sigv4.rs @@ -0,0 +1,193 @@ +use hmac::{Hmac, Mac}; +use lru::LruCache; +use parking_lot::Mutex; +use percent_encoding::{percent_encode, AsciiSet, NON_ALPHANUMERIC}; +use pyo3::prelude::*; +use sha2::{Digest, Sha256}; +use std::num::NonZeroUsize; +use std::sync::LazyLock; +use std::time::Instant; + +type HmacSha256 = Hmac; + +struct CacheEntry { + key: Vec, + created: Instant, +} + +static SIGNING_KEY_CACHE: LazyLock>> = + LazyLock::new(|| Mutex::new(LruCache::new(NonZeroUsize::new(256).unwrap()))); + +const CACHE_TTL_SECS: u64 = 60; + +const AWS_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC + .remove(b'-') + .remove(b'_') + .remove(b'.') + .remove(b'~'); + +fn hmac_sha256(key: &[u8], msg: &[u8]) -> Vec { + let mut mac = HmacSha256::new_from_slice(key).expect("HMAC key length is always valid"); + mac.update(msg); + mac.finalize().into_bytes().to_vec() +} + +fn sha256_hex(data: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(data); + hex::encode(hasher.finalize()) +} + +fn aws_uri_encode(input: &str) -> String { + percent_encode(input.as_bytes(), AWS_ENCODE_SET).to_string() +} + +fn derive_signing_key_cached( + secret_key: &str, + date_stamp: &str, + region: &str, + service: &str, +) -> Vec { + let cache_key = ( + secret_key.to_owned(), + date_stamp.to_owned(), + region.to_owned(), + service.to_owned(), + ); + + { + let mut cache = SIGNING_KEY_CACHE.lock(); + if let Some(entry) = cache.get(&cache_key) { + if entry.created.elapsed().as_secs() < CACHE_TTL_SECS { + return entry.key.clone(); + } + cache.pop(&cache_key); + } + } + + let k_date = hmac_sha256(format!("AWS4{}", secret_key).as_bytes(), date_stamp.as_bytes()); + let k_region = hmac_sha256(&k_date, region.as_bytes()); + let k_service = hmac_sha256(&k_region, service.as_bytes()); + let k_signing = hmac_sha256(&k_service, b"aws4_request"); + + { + let mut cache = SIGNING_KEY_CACHE.lock(); + cache.put( + cache_key, + CacheEntry { + key: k_signing.clone(), + created: Instant::now(), + }, + ); + } + + k_signing +} + +fn constant_time_compare_inner(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + let mut result: u8 = 0; + for (x, y) in a.iter().zip(b.iter()) { + result |= x ^ y; + } + result == 0 +} + +#[pyfunction] +pub fn verify_sigv4_signature( + method: &str, + canonical_uri: &str, + query_params: Vec<(String, String)>, + signed_headers_str: &str, + header_values: Vec<(String, String)>, + payload_hash: &str, + amz_date: &str, + date_stamp: &str, + region: &str, + service: &str, + secret_key: &str, + provided_signature: &str, +) -> bool { + let mut sorted_params = query_params; + sorted_params.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1))); + + let canonical_query_string = sorted_params + .iter() + .map(|(k, v)| format!("{}={}", aws_uri_encode(k), aws_uri_encode(v))) + .collect::>() + .join("&"); + + let mut canonical_headers = String::new(); + for (name, value) in &header_values { + let lower_name = name.to_lowercase(); + let normalized = value.split_whitespace().collect::>().join(" "); + let final_value = if lower_name == "expect" && normalized.is_empty() { + "100-continue" + } else { + &normalized + }; + canonical_headers.push_str(&lower_name); + canonical_headers.push(':'); + canonical_headers.push_str(final_value); + canonical_headers.push('\n'); + } + + let canonical_request = format!( + "{}\n{}\n{}\n{}\n{}\n{}", + method, canonical_uri, canonical_query_string, canonical_headers, signed_headers_str, payload_hash + ); + + let credential_scope = format!("{}/{}/{}/aws4_request", date_stamp, region, service); + let cr_hash = sha256_hex(canonical_request.as_bytes()); + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + amz_date, credential_scope, cr_hash + ); + + let signing_key = derive_signing_key_cached(secret_key, date_stamp, region, service); + let calculated = hmac_sha256(&signing_key, string_to_sign.as_bytes()); + let calculated_hex = hex::encode(&calculated); + + constant_time_compare_inner(calculated_hex.as_bytes(), provided_signature.as_bytes()) +} + +#[pyfunction] +pub fn derive_signing_key( + secret_key: &str, + date_stamp: &str, + region: &str, + service: &str, +) -> Vec { + derive_signing_key_cached(secret_key, date_stamp, region, service) +} + +#[pyfunction] +pub fn compute_signature(signing_key: &[u8], string_to_sign: &str) -> String { + let sig = hmac_sha256(signing_key, string_to_sign.as_bytes()); + hex::encode(sig) +} + +#[pyfunction] +pub fn build_string_to_sign( + amz_date: &str, + credential_scope: &str, + canonical_request: &str, +) -> String { + let cr_hash = sha256_hex(canonical_request.as_bytes()); + format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + amz_date, credential_scope, cr_hash + ) +} + +#[pyfunction] +pub fn constant_time_compare(a: &str, b: &str) -> bool { + constant_time_compare_inner(a.as_bytes(), b.as_bytes()) +} + +#[pyfunction] +pub fn clear_signing_key_cache() { + SIGNING_KEY_CACHE.lock().clear(); +} diff --git a/myfsio_core/src/validation.rs b/myfsio_core/src/validation.rs new file mode 100644 index 0000000..06b4e84 --- /dev/null +++ b/myfsio_core/src/validation.rs @@ -0,0 +1,149 @@ +use pyo3::prelude::*; +use std::sync::LazyLock; +use unicode_normalization::UnicodeNormalization; + +const WINDOWS_RESERVED: &[&str] = &[ + "CON", "PRN", "AUX", "NUL", "COM0", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", + "COM8", "COM9", "LPT0", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", + "LPT9", +]; + +const WINDOWS_ILLEGAL_CHARS: &[char] = &['<', '>', ':', '"', '/', '\\', '|', '?', '*']; + +const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"]; +const SYSTEM_ROOT: &str = ".myfsio.sys"; + +static IP_REGEX: LazyLock = + LazyLock::new(|| regex::Regex::new(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$").unwrap()); + +#[pyfunction] +#[pyo3(signature = (object_key, max_length_bytes=1024, is_windows=false, reserved_prefixes=None))] +pub fn validate_object_key( + object_key: &str, + max_length_bytes: usize, + is_windows: bool, + reserved_prefixes: Option>, +) -> PyResult> { + if object_key.is_empty() { + return Ok(Some("Object key required".to_string())); + } + + if object_key.contains('\0') { + return Ok(Some("Object key contains null bytes".to_string())); + } + + let normalized: String = object_key.nfc().collect(); + + if normalized.len() > max_length_bytes { + return Ok(Some(format!( + "Object key exceeds maximum length of {} bytes", + max_length_bytes + ))); + } + + if normalized.starts_with('/') || normalized.starts_with('\\') { + return Ok(Some("Object key cannot start with a slash".to_string())); + } + + let parts: Vec<&str> = if cfg!(windows) || is_windows { + normalized.split(['/', '\\']).collect() + } else { + normalized.split('/').collect() + }; + + for part in &parts { + if part.is_empty() { + continue; + } + + if *part == ".." { + return Ok(Some( + "Object key contains parent directory references".to_string(), + )); + } + + if *part == "." { + return Ok(Some("Object key contains invalid segments".to_string())); + } + + if part.chars().any(|c| (c as u32) < 32) { + return Ok(Some( + "Object key contains control characters".to_string(), + )); + } + + if is_windows { + if part.chars().any(|c| WINDOWS_ILLEGAL_CHARS.contains(&c)) { + return Ok(Some( + "Object key contains characters not supported on Windows filesystems" + .to_string(), + )); + } + if part.ends_with(' ') || part.ends_with('.') { + return Ok(Some( + "Object key segments cannot end with spaces or periods on Windows".to_string(), + )); + } + let trimmed = part.trim_end_matches(['.', ' ']).to_uppercase(); + if WINDOWS_RESERVED.contains(&trimmed.as_str()) { + return Ok(Some(format!("Invalid filename segment: {}", part))); + } + } + } + + let non_empty_parts: Vec<&str> = parts.iter().filter(|p| !p.is_empty()).copied().collect(); + if let Some(top) = non_empty_parts.first() { + if INTERNAL_FOLDERS.contains(top) || *top == SYSTEM_ROOT { + return Ok(Some("Object key uses a reserved prefix".to_string())); + } + + if let Some(ref prefixes) = reserved_prefixes { + for prefix in prefixes { + if *top == prefix.as_str() { + return Ok(Some("Object key uses a reserved prefix".to_string())); + } + } + } + } + + Ok(None) +} + +#[pyfunction] +pub fn validate_bucket_name(bucket_name: &str) -> Option { + let len = bucket_name.len(); + if len < 3 || len > 63 { + return Some("Bucket name must be between 3 and 63 characters".to_string()); + } + + let bytes = bucket_name.as_bytes(); + if !bytes[0].is_ascii_lowercase() && !bytes[0].is_ascii_digit() { + return Some( + "Bucket name must start and end with a lowercase letter or digit".to_string(), + ); + } + if !bytes[len - 1].is_ascii_lowercase() && !bytes[len - 1].is_ascii_digit() { + return Some( + "Bucket name must start and end with a lowercase letter or digit".to_string(), + ); + } + + for &b in bytes { + if !b.is_ascii_lowercase() && !b.is_ascii_digit() && b != b'.' && b != b'-' { + return Some( + "Bucket name can only contain lowercase letters, digits, dots, and hyphens" + .to_string(), + ); + } + } + + if bucket_name.contains("..") { + return Some("Bucket name must not contain consecutive periods".to_string()); + } + + if IP_REGEX.is_match(bucket_name) { + return Some("Bucket name must not be formatted as an IP address".to_string()); + } + + None +} diff --git a/static/css/main.css b/static/css/main.css index 0ab8050..9b38cd3 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -1151,17 +1151,123 @@ html.sidebar-will-collapse .sidebar-user { } .iam-user-card { - border: 1px solid var(--myfsio-card-border); - border-radius: 0.75rem; - transition: box-shadow 0.2s ease, transform 0.2s ease; + position: relative; + border: 1px solid var(--myfsio-card-border) !important; + border-radius: 1rem !important; + overflow: hidden; + transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1); +} + +.iam-user-card::before { + content: ''; + position: absolute; + top: 0; + left: 0; + right: 0; + height: 4px; + background: linear-gradient(90deg, #3b82f6, #8b5cf6); + opacity: 0; + transition: opacity 0.2s ease; } .iam-user-card:hover { - box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1); + transform: translateY(-2px); + box-shadow: 0 8px 24px -4px rgba(0, 0, 0, 0.12), 0 4px 8px -4px rgba(0, 0, 0, 0.08); + border-color: var(--myfsio-accent) !important; +} + +.iam-user-card:hover::before { + opacity: 1; } [data-theme='dark'] .iam-user-card:hover { - box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3); + box-shadow: 0 8px 24px -4px rgba(0, 0, 0, 0.4), 0 4px 8px -4px rgba(0, 0, 0, 0.3); +} + +.iam-admin-card::before { + background: linear-gradient(90deg, #f59e0b, #ef4444); +} + +.iam-role-badge { + display: inline-flex; + align-items: center; + padding: 0.25em 0.65em; + border-radius: 999px; + font-size: 0.7rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.03em; +} + +.iam-role-admin { + background: rgba(245, 158, 11, 0.15); + color: #d97706; +} + +[data-theme='dark'] .iam-role-admin { + background: rgba(245, 158, 11, 0.25); + color: #fbbf24; +} + +.iam-role-user { + background: rgba(59, 130, 246, 0.12); + color: #2563eb; +} + +[data-theme='dark'] .iam-role-user { + background: rgba(59, 130, 246, 0.2); + color: #60a5fa; +} + +.iam-perm-badge { + display: inline-flex; + align-items: center; + gap: 0.25rem; + padding: 0.3em 0.6em; + border-radius: 999px; + font-size: 0.75rem; + font-weight: 500; + background: rgba(59, 130, 246, 0.08); + color: var(--myfsio-text); + border: 1px solid rgba(59, 130, 246, 0.15); +} + +[data-theme='dark'] .iam-perm-badge { + background: rgba(59, 130, 246, 0.15); + border-color: rgba(59, 130, 246, 0.25); +} + +.iam-copy-key { + display: inline-flex; + align-items: center; + justify-content: center; + width: 22px; + height: 22px; + padding: 0; + border: none; + background: transparent; + color: var(--myfsio-muted); + border-radius: 4px; + cursor: pointer; + transition: all 0.15s ease; + flex-shrink: 0; +} + +.iam-copy-key:hover { + background: var(--myfsio-hover-bg); + color: var(--myfsio-text); +} + +.iam-no-results { + text-align: center; + padding: 2rem 1rem; + color: var(--myfsio-muted); +} + +@media (max-width: 768px) { + .iam-user-card:hover { + transform: none; + } } .user-avatar-lg { @@ -2819,6 +2925,112 @@ body:has(.login-card) .main-wrapper { padding-top: 0 !important; } +.context-menu { + position: fixed; + z-index: 1060; + min-width: 180px; + background: var(--myfsio-card-bg); + border: 1px solid var(--myfsio-card-border); + border-radius: 0.5rem; + box-shadow: 0 10px 25px -5px rgba(0, 0, 0, 0.15), 0 8px 10px -6px rgba(0, 0, 0, 0.1); + padding: 0.25rem 0; + font-size: 0.875rem; +} + +[data-theme='dark'] .context-menu { + box-shadow: 0 10px 25px -5px rgba(0, 0, 0, 0.4), 0 8px 10px -6px rgba(0, 0, 0, 0.3); +} + +.context-menu-item { + display: flex; + align-items: center; + gap: 0.625rem; + padding: 0.5rem 0.875rem; + color: var(--myfsio-text); + cursor: pointer; + transition: background-color 0.1s ease; + border: none; + background: none; + width: 100%; + text-align: left; + font-size: inherit; +} + +.context-menu-item:hover { + background-color: var(--myfsio-hover-bg); +} + +.context-menu-item.text-danger:hover { + background-color: rgba(239, 68, 68, 0.1); +} + +.context-menu-divider { + height: 1px; + background: var(--myfsio-card-border); + margin: 0.25rem 0; +} + +.context-menu-shortcut { + margin-left: auto; + font-size: 0.75rem; + color: var(--myfsio-muted); +} + +.kbd-shortcuts-list { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.kbd-shortcuts-list .shortcut-row { + display: flex; + align-items: center; + justify-content: space-between; + padding: 0.375rem 0; +} + +.kbd-shortcuts-list kbd { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 1.75rem; + padding: 0.2rem 0.5rem; + font-family: inherit; + font-size: 0.75rem; + font-weight: 600; + background: var(--myfsio-preview-bg); + border: 1px solid var(--myfsio-card-border); + border-radius: 0.25rem; + box-shadow: 0 1px 0 1px rgba(0, 0, 0, 0.05); + color: var(--myfsio-text); +} + +[data-theme='dark'] .kbd-shortcuts-list kbd { + background: rgba(255, 255, 255, 0.1); + box-shadow: 0 1px 0 1px rgba(0, 0, 0, 0.2); +} + +.sort-dropdown .dropdown-item.active, +.sort-dropdown .dropdown-item:active { + background-color: var(--myfsio-hover-bg); + color: var(--myfsio-text); +} + +.sort-dropdown .dropdown-item { + font-size: 0.875rem; + padding: 0.375rem 1rem; +} + +@media (prefers-reduced-motion: reduce) { + *, + *::before, + *::after { + animation-duration: 0.01ms !important; + animation-iteration-count: 1 !important; + transition-duration: 0.01ms !important; + } +} + @media print { .sidebar, .mobile-header { diff --git a/static/js/bucket-detail-main.js b/static/js/bucket-detail-main.js index c643b09..3ff9871 100644 --- a/static/js/bucket-detail-main.js +++ b/static/js/bucket-detail-main.js @@ -162,6 +162,8 @@ let isLoadingObjects = false; let hasMoreObjects = false; let currentFilterTerm = ''; + let currentSortField = 'name'; + let currentSortDir = 'asc'; let pageSize = 5000; let currentPrefix = ''; let allObjects = []; @@ -348,14 +350,18 @@ const currentInputs = { objectCount: allObjects.length, prefix: currentPrefix, - filterTerm: currentFilterTerm + filterTerm: currentFilterTerm, + sortField: currentSortField, + sortDir: currentSortDir }; if (!forceRecompute && memoizedVisibleItems !== null && memoizedInputs.objectCount === currentInputs.objectCount && memoizedInputs.prefix === currentInputs.prefix && - memoizedInputs.filterTerm === currentInputs.filterTerm) { + memoizedInputs.filterTerm === currentInputs.filterTerm && + memoizedInputs.sortField === currentInputs.sortField && + memoizedInputs.sortDir === currentInputs.sortDir) { return memoizedVisibleItems; } @@ -394,9 +400,19 @@ items.sort((a, b) => { if (a.type === 'folder' && b.type === 'file') return -1; if (a.type === 'file' && b.type === 'folder') return 1; - const aKey = a.type === 'folder' ? a.path : a.data.key; - const bKey = b.type === 'folder' ? b.path : b.data.key; - return aKey.localeCompare(bKey); + if (a.type === 'folder' && b.type === 'folder') { + return a.path.localeCompare(b.path); + } + const dir = currentSortDir === 'asc' ? 1 : -1; + if (currentSortField === 'size') { + return (a.data.size - b.data.size) * dir; + } + if (currentSortField === 'date') { + const aTime = new Date(a.data.lastModified || a.data.last_modified || 0).getTime(); + const bTime = new Date(b.data.lastModified || b.data.last_modified || 0).getTime(); + return (aTime - bTime) * dir; + } + return a.data.key.localeCompare(b.data.key) * dir; }); memoizedVisibleItems = items; @@ -2034,6 +2050,128 @@ refreshVirtualList(); }); + document.querySelectorAll('[data-sort-field]').forEach(el => { + el.addEventListener('click', (e) => { + e.preventDefault(); + const field = el.dataset.sortField; + const dir = el.dataset.sortDir || 'asc'; + currentSortField = field; + currentSortDir = dir; + document.querySelectorAll('[data-sort-field]').forEach(s => s.classList.remove('active')); + el.classList.add('active'); + var label = document.getElementById('sort-dropdown-label'); + if (label) label.textContent = el.textContent.trim(); + refreshVirtualList(); + }); + }); + + document.addEventListener('keydown', (e) => { + if (e.target.tagName === 'INPUT' || e.target.tagName === 'TEXTAREA' || e.target.tagName === 'SELECT' || e.target.isContentEditable) return; + + if (e.key === '/' && !e.ctrlKey && !e.metaKey) { + e.preventDefault(); + document.getElementById('object-search')?.focus(); + } + + if (e.key === '?' && !e.ctrlKey && !e.metaKey) { + e.preventDefault(); + var kbModal = document.getElementById('keyboardShortcutsModal'); + if (kbModal) { + var instance = bootstrap.Modal.getOrCreateInstance(kbModal); + instance.toggle(); + } + } + + if (e.key === 'Escape') { + var searchInput = document.getElementById('object-search'); + if (searchInput && document.activeElement === searchInput) { + searchInput.value = ''; + currentFilterTerm = ''; + refreshVirtualList(); + searchInput.blur(); + } + } + + if (e.key === 'Delete' && !e.ctrlKey && !e.metaKey) { + if (selectedRows.size > 0 && bulkDeleteButton && !bulkDeleteButton.disabled) { + bulkDeleteButton.click(); + } + } + + if (e.key === 'a' && (e.ctrlKey || e.metaKey)) { + if (visibleItems.length > 0 && selectAllCheckbox) { + e.preventDefault(); + selectAllCheckbox.checked = true; + selectAllCheckbox.dispatchEvent(new Event('change')); + } + } + }); + + const ctxMenu = document.getElementById('objectContextMenu'); + let ctxTargetRow = null; + + const hideContextMenu = () => { + if (ctxMenu) ctxMenu.classList.add('d-none'); + ctxTargetRow = null; + }; + + if (ctxMenu) { + document.addEventListener('click', hideContextMenu); + document.addEventListener('contextmenu', (e) => { + const row = e.target.closest('[data-object-row]'); + if (!row) { hideContextMenu(); return; } + e.preventDefault(); + ctxTargetRow = row; + + const x = Math.min(e.clientX, window.innerWidth - 200); + const y = Math.min(e.clientY, window.innerHeight - 200); + ctxMenu.style.left = x + 'px'; + ctxMenu.style.top = y + 'px'; + ctxMenu.classList.remove('d-none'); + }); + + ctxMenu.querySelectorAll('[data-ctx-action]').forEach(btn => { + btn.addEventListener('click', () => { + if (!ctxTargetRow) return; + const action = btn.dataset.ctxAction; + const key = ctxTargetRow.dataset.key; + const bucket = objectsContainer?.dataset.bucket || ''; + + if (action === 'download') { + const url = ctxTargetRow.dataset.downloadUrl; + if (url) window.open(url, '_blank'); + } else if (action === 'copy-path') { + const s3Path = 's3://' + bucket + '/' + key; + if (navigator.clipboard) { + navigator.clipboard.writeText(s3Path).then(() => { + if (window.showToast) window.showToast('Copied: ' + s3Path, 'Copied', 'success'); + }); + } + } else if (action === 'presign') { + selectRow(ctxTargetRow); + presignLink.value = ''; + presignModal?.show(); + requestPresignedUrl(); + } else if (action === 'delete') { + const deleteEndpoint = ctxTargetRow.dataset.deleteEndpoint; + if (deleteEndpoint) { + selectRow(ctxTargetRow); + const deleteModalEl = document.getElementById('deleteObjectModal'); + const deleteModal = deleteModalEl ? bootstrap.Modal.getOrCreateInstance(deleteModalEl) : null; + const deleteObjectForm = document.getElementById('deleteObjectForm'); + const deleteObjectKey = document.getElementById('deleteObjectKey'); + if (deleteModal && deleteObjectForm) { + deleteObjectForm.setAttribute('action', deleteEndpoint); + if (deleteObjectKey) deleteObjectKey.textContent = key; + deleteModal.show(); + } + } + } + hideContextMenu(); + }); + }); + } + refreshVersionsButton?.addEventListener('click', () => { if (!activeRow) { versionList.innerHTML = '

Select an object to view versions.

'; diff --git a/static/js/connections-management.js b/static/js/connections-management.js index e58b0be..8d22a6c 100644 --- a/static/js/connections-management.js +++ b/static/js/connections-management.js @@ -78,7 +78,7 @@ window.ConnectionsManagement = (function() { try { var controller = new AbortController(); - var timeoutId = setTimeout(function() { controller.abort(); }, 15000); + var timeoutId = setTimeout(function() { controller.abort(); }, 10000); var response = await fetch(endpoints.healthTemplate.replace('CONNECTION_ID', connectionId), { signal: controller.signal @@ -147,7 +147,7 @@ window.ConnectionsManagement = (function() { '' + '' + + '' + '' + '
' + '' + '
    ' + - '
    • ' + - '
    • ' + '
    • ' + - '
    • ' + '
' + '
' + '
Bucket Permissions
' + - '
' + policyBadges + '
' + - '' + ''; } @@ -342,6 +388,13 @@ window.IAMManagement = (function() { policyModal.show(); }); } + + var copyBtn = cardElement.querySelector('[data-copy-access-key]'); + if (copyBtn) { + copyBtn.addEventListener('click', function() { + copyAccessKey(copyBtn); + }); + } } function updateUserCount() { @@ -442,17 +495,33 @@ window.IAMManagement = (function() { var userCard = document.querySelector('[data-access-key="' + key + '"]'); if (userCard) { - var badgeContainer = userCard.closest('.iam-user-card').querySelector('.d-flex.flex-wrap.gap-1'); + var cardEl = userCard.closest('.iam-user-card'); + var badgeContainer = cardEl ? cardEl.querySelector('[data-policy-badges]') : null; if (badgeContainer && data.policies) { var badges = data.policies.map(function(p) { - return '' + + var bl = getBucketLabel(p.bucket); + var pl = getPermissionLevel(p.actions); + return '' + '' + '' + - '' + window.UICore.escapeHtml(p.bucket) + - '(' + (p.actions.includes('*') ? 'full' : p.actions.length) + ')'; + '' + window.UICore.escapeHtml(bl) + ' · ' + window.UICore.escapeHtml(pl) + ''; }).join(''); badgeContainer.innerHTML = badges || 'No policies'; } + if (cardEl) { + var nowAdmin = isAdminUser(data.policies); + cardEl.classList.toggle('iam-admin-card', nowAdmin); + var roleBadgeEl = cardEl.querySelector('[data-role-badge]'); + if (roleBadgeEl) { + if (nowAdmin) { + roleBadgeEl.className = 'iam-role-badge iam-role-admin'; + roleBadgeEl.textContent = 'Admin'; + } else { + roleBadgeEl.className = 'iam-role-badge iam-role-user'; + roleBadgeEl.textContent = 'User'; + } + } + } } var userIndex = users.findIndex(function(u) { return u.access_key === key; }); @@ -485,6 +554,10 @@ window.IAMManagement = (function() { nameEl.textContent = newName; nameEl.title = newName; } + var itemWrapper = card.closest('.iam-user-item'); + if (itemWrapper) { + itemWrapper.setAttribute('data-display-name', newName.toLowerCase()); + } } } @@ -539,6 +612,52 @@ window.IAMManagement = (function() { } } + function setupSearch() { + var searchInput = document.getElementById('iam-user-search'); + if (!searchInput) return; + + searchInput.addEventListener('input', function() { + var query = searchInput.value.toLowerCase().trim(); + var items = document.querySelectorAll('.iam-user-item'); + var noResults = document.getElementById('iam-no-results'); + var visibleCount = 0; + + items.forEach(function(item) { + var name = item.getAttribute('data-display-name') || ''; + var key = item.getAttribute('data-access-key-filter') || ''; + var matches = !query || name.indexOf(query) >= 0 || key.indexOf(query) >= 0; + item.classList.toggle('d-none', !matches); + if (matches) visibleCount++; + }); + + if (noResults) { + noResults.classList.toggle('d-none', visibleCount > 0); + } + }); + } + + function copyAccessKey(btn) { + var key = btn.getAttribute('data-copy-access-key'); + if (!key) return; + var originalHtml = btn.innerHTML; + navigator.clipboard.writeText(key).then(function() { + btn.innerHTML = ''; + btn.style.color = '#22c55e'; + setTimeout(function() { + btn.innerHTML = originalHtml; + btn.style.color = ''; + }, 1200); + }).catch(function() {}); + } + + function setupCopyAccessKeyButtons() { + document.querySelectorAll('[data-copy-access-key]').forEach(function(btn) { + btn.addEventListener('click', function() { + copyAccessKey(btn); + }); + }); + } + return { init: init }; diff --git a/static/js/ui-core.js b/static/js/ui-core.js index a69d7ef..4c20162 100644 --- a/static/js/ui-core.js +++ b/static/js/ui-core.js @@ -191,6 +191,10 @@ window.UICore = (function() { } }); + window.addEventListener('beforeunload', function() { + pollingManager.stopAll(); + }); + return { getCsrfToken: getCsrfToken, formatBytes: formatBytes, diff --git a/templates/bucket_detail.html b/templates/bucket_detail.html index 9289019..14f3406 100644 --- a/templates/bucket_detail.html +++ b/templates/bucket_detail.html @@ -100,8 +100,26 @@ Upload +
+ +
    +
    • +
    • +
    • +
    • +
    • +
    • +
    • +
    • +
+
- +
{% endif %} + {% if website_domains %} +
+ + {% for domain in website_domains %} +
+ + + + + connected + + {{ domain }} +
+ {% endfor %} +
+ {% elif website_config %} +
+ +

No domains mapped to this bucket. Manage domains

+
+ {% endif %} + {% if can_manage_website %}
@@ -2663,6 +2703,63 @@ + +
+ + + +
+ +
+ + {% endblock %} {% block extra_scripts %} diff --git a/templates/buckets.html b/templates/buckets.html index 13ea928..ad24574 100644 --- a/templates/buckets.html +++ b/templates/buckets.html @@ -89,6 +89,14 @@ {% endfor %} +
+
+ + + +

No buckets match your filter.

+
+