6 Commits

518 changed files with 5545 additions and 671 deletions

View File

@@ -6,14 +6,26 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
WORKDIR /app WORKDIR /app
RUN apt-get update \ RUN apt-get update \
&& apt-get install -y --no-install-recommends build-essential \ && apt-get install -y --no-install-recommends build-essential curl \
&& curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
ENV PATH="/root/.cargo/bin:${PATH}"
COPY requirements.txt ./ COPY requirements.txt ./
RUN pip install --no-cache-dir -r requirements.txt RUN pip install --no-cache-dir -r requirements.txt
COPY . . COPY . .
RUN pip install --no-cache-dir maturin \
&& cd myfsio_core \
&& maturin build --release \
&& pip install target/wheels/*.whl \
&& cd .. \
&& rm -rf myfsio_core/target \
&& pip uninstall -y maturin \
&& rustup self uninstall -y
RUN chmod +x docker-entrypoint.sh RUN chmod +x docker-entrypoint.sh
RUN mkdir -p /app/data \ RUN mkdir -p /app/data \

View File

@@ -1,6 +1,8 @@
from __future__ import annotations from __future__ import annotations
import html as html_module
import logging import logging
import mimetypes
import shutil import shutil
import sys import sys
import time import time
@@ -10,7 +12,7 @@ from pathlib import Path
from datetime import timedelta from datetime import timedelta
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
from flask import Flask, g, has_request_context, redirect, render_template, request, url_for from flask import Flask, Response, g, has_request_context, redirect, render_template, request, url_for
from flask_cors import CORS from flask_cors import CORS
from flask_wtf.csrf import CSRFError from flask_wtf.csrf import CSRFError
from werkzeug.middleware.proxy_fix import ProxyFix from werkzeug.middleware.proxy_fix import ProxyFix
@@ -32,8 +34,9 @@ from .object_lock import ObjectLockService
from .replication import ReplicationManager from .replication import ReplicationManager
from .secret_store import EphemeralSecretStore from .secret_store import EphemeralSecretStore
from .site_registry import SiteRegistry, SiteInfo from .site_registry import SiteRegistry, SiteInfo
from .storage import ObjectStorage from .storage import ObjectStorage, StorageError
from .version import get_version from .version import get_version
from .website_domains import WebsiteDomainStore
def _migrate_config_file(active_path: Path, legacy_paths: List[Path]) -> Path: def _migrate_config_file(active_path: Path, legacy_paths: List[Path]) -> Path:
@@ -223,6 +226,12 @@ def create_app(
app.extensions["access_logging"] = access_logging_service app.extensions["access_logging"] = access_logging_service
app.extensions["site_registry"] = site_registry app.extensions["site_registry"] = site_registry
website_domains_store = None
if app.config.get("WEBSITE_HOSTING_ENABLED", False):
website_domains_path = config_dir / "website_domains.json"
website_domains_store = WebsiteDomainStore(website_domains_path)
app.extensions["website_domains"] = website_domains_store
from .s3_client import S3ProxyClient from .s3_client import S3ProxyClient
api_base = app.config.get("API_BASE_URL") or "http://127.0.0.1:5000" api_base = app.config.get("API_BASE_URL") or "http://127.0.0.1:5000"
app.extensions["s3_proxy"] = S3ProxyClient( app.extensions["s3_proxy"] = S3ProxyClient(
@@ -472,6 +481,128 @@ def _configure_logging(app: Flask) -> None:
extra={"path": request.path, "method": request.method, "remote_addr": request.remote_addr}, extra={"path": request.path, "method": request.method, "remote_addr": request.remote_addr},
) )
@app.before_request
def _maybe_serve_website():
if not app.config.get("WEBSITE_HOSTING_ENABLED"):
return None
if request.method not in {"GET", "HEAD"}:
return None
host = request.host
if ":" in host:
host = host.rsplit(":", 1)[0]
host = host.lower()
store = app.extensions.get("website_domains")
if not store:
return None
bucket = store.get_bucket(host)
if not bucket:
return None
storage = app.extensions["object_storage"]
if not storage.bucket_exists(bucket):
return _website_error_response(404, "Not Found")
website_config = storage.get_bucket_website(bucket)
if not website_config:
return _website_error_response(404, "Not Found")
index_doc = website_config.get("index_document", "index.html")
error_doc = website_config.get("error_document")
req_path = request.path.lstrip("/")
if not req_path or req_path.endswith("/"):
object_key = req_path + index_doc
else:
object_key = req_path
try:
obj_path = storage.get_object_path(bucket, object_key)
except (StorageError, OSError):
if object_key == req_path:
try:
obj_path = storage.get_object_path(bucket, req_path + "/" + index_doc)
object_key = req_path + "/" + index_doc
except (StorageError, OSError):
return _serve_website_error(storage, bucket, error_doc, 404)
else:
return _serve_website_error(storage, bucket, error_doc, 404)
content_type = mimetypes.guess_type(object_key)[0] or "application/octet-stream"
is_encrypted = False
try:
metadata = storage.get_object_metadata(bucket, object_key)
is_encrypted = "x-amz-server-side-encryption" in metadata
except (StorageError, OSError):
pass
if request.method == "HEAD":
response = Response(status=200)
if is_encrypted and hasattr(storage, "get_object_data"):
try:
data, _ = storage.get_object_data(bucket, object_key)
response.headers["Content-Length"] = len(data)
except (StorageError, OSError):
return _website_error_response(500, "Internal Server Error")
else:
try:
stat = obj_path.stat()
response.headers["Content-Length"] = stat.st_size
except OSError:
return _website_error_response(500, "Internal Server Error")
response.headers["Content-Type"] = content_type
return response
if is_encrypted and hasattr(storage, "get_object_data"):
try:
data, _ = storage.get_object_data(bucket, object_key)
response = Response(data, mimetype=content_type)
response.headers["Content-Length"] = len(data)
return response
except (StorageError, OSError):
return _website_error_response(500, "Internal Server Error")
def _stream(file_path):
with file_path.open("rb") as f:
while True:
chunk = f.read(65536)
if not chunk:
break
yield chunk
try:
stat = obj_path.stat()
response = Response(_stream(obj_path), mimetype=content_type, direct_passthrough=True)
response.headers["Content-Length"] = stat.st_size
return response
except OSError:
return _website_error_response(500, "Internal Server Error")
def _serve_website_error(storage, bucket, error_doc_key, status_code):
if not error_doc_key:
return _website_error_response(status_code, "Not Found" if status_code == 404 else "Error")
try:
obj_path = storage.get_object_path(bucket, error_doc_key)
except (StorageError, OSError):
return _website_error_response(status_code, "Not Found")
content_type = mimetypes.guess_type(error_doc_key)[0] or "text/html"
is_encrypted = False
try:
metadata = storage.get_object_metadata(bucket, error_doc_key)
is_encrypted = "x-amz-server-side-encryption" in metadata
except (StorageError, OSError):
pass
if is_encrypted and hasattr(storage, "get_object_data"):
try:
data, _ = storage.get_object_data(bucket, error_doc_key)
response = Response(data, status=status_code, mimetype=content_type)
response.headers["Content-Length"] = len(data)
return response
except (StorageError, OSError):
return _website_error_response(status_code, "Not Found")
try:
data = obj_path.read_bytes()
response = Response(data, status=status_code, mimetype=content_type)
response.headers["Content-Length"] = len(data)
return response
except OSError:
return _website_error_response(status_code, "Not Found")
def _website_error_response(status_code, message):
safe_msg = html_module.escape(str(message))
safe_code = html_module.escape(str(status_code))
body = f"<html><head><title>{safe_code} {safe_msg}</title></head><body><h1>{safe_code} {safe_msg}</h1></body></html>"
return Response(body, status=status_code, mimetype="text/html")
@app.after_request @app.after_request
def _log_request_end(response): def _log_request_end(response):
duration_ms = 0.0 duration_ms = 0.0

View File

@@ -17,6 +17,7 @@ from .extensions import limiter
from .iam import IamError, Principal from .iam import IamError, Principal
from .replication import ReplicationManager from .replication import ReplicationManager
from .site_registry import PeerSite, SiteInfo, SiteRegistry from .site_registry import PeerSite, SiteInfo, SiteRegistry
from .website_domains import WebsiteDomainStore, normalize_domain, is_valid_domain
def _is_safe_url(url: str, allow_internal: bool = False) -> bool: def _is_safe_url(url: str, allow_internal: bool = False) -> bool:
@@ -673,3 +674,105 @@ def check_bidirectional_status(site_id: str):
result["is_fully_configured"] = len(error_issues) == 0 and len(local_bidir_rules) > 0 result["is_fully_configured"] = len(error_issues) == 0 and len(local_bidir_rules) > 0
return jsonify(result) return jsonify(result)
def _website_domains() -> WebsiteDomainStore:
return current_app.extensions["website_domains"]
def _storage():
return current_app.extensions["object_storage"]
@admin_api_bp.route("/website-domains", methods=["GET"])
@limiter.limit(lambda: _get_admin_rate_limit())
def list_website_domains():
principal, error = _require_admin()
if error:
return error
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
return _json_error("InvalidRequest", "Website hosting is not enabled", 400)
return jsonify(_website_domains().list_all())
@admin_api_bp.route("/website-domains", methods=["POST"])
@limiter.limit(lambda: _get_admin_rate_limit())
def create_website_domain():
principal, error = _require_admin()
if error:
return error
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
return _json_error("InvalidRequest", "Website hosting is not enabled", 400)
payload = request.get_json(silent=True) or {}
domain = normalize_domain(payload.get("domain") or "")
bucket = (payload.get("bucket") or "").strip()
if not domain:
return _json_error("ValidationError", "domain is required", 400)
if not is_valid_domain(domain):
return _json_error("ValidationError", f"Invalid domain: '{domain}'", 400)
if not bucket:
return _json_error("ValidationError", "bucket is required", 400)
storage = _storage()
if not storage.bucket_exists(bucket):
return _json_error("NoSuchBucket", f"Bucket '{bucket}' does not exist", 404)
store = _website_domains()
existing = store.get_bucket(domain)
if existing:
return _json_error("Conflict", f"Domain '{domain}' is already mapped to bucket '{existing}'", 409)
store.set_mapping(domain, bucket)
logger.info("Website domain mapping created: %s -> %s", domain, bucket)
return jsonify({"domain": domain, "bucket": bucket}), 201
@admin_api_bp.route("/website-domains/<domain>", methods=["GET"])
@limiter.limit(lambda: _get_admin_rate_limit())
def get_website_domain(domain: str):
principal, error = _require_admin()
if error:
return error
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
return _json_error("InvalidRequest", "Website hosting is not enabled", 400)
domain = normalize_domain(domain)
bucket = _website_domains().get_bucket(domain)
if not bucket:
return _json_error("NotFound", f"No mapping found for domain '{domain}'", 404)
return jsonify({"domain": domain, "bucket": bucket})
@admin_api_bp.route("/website-domains/<domain>", methods=["PUT"])
@limiter.limit(lambda: _get_admin_rate_limit())
def update_website_domain(domain: str):
principal, error = _require_admin()
if error:
return error
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
return _json_error("InvalidRequest", "Website hosting is not enabled", 400)
domain = normalize_domain(domain)
payload = request.get_json(silent=True) or {}
bucket = (payload.get("bucket") or "").strip()
if not bucket:
return _json_error("ValidationError", "bucket is required", 400)
storage = _storage()
if not storage.bucket_exists(bucket):
return _json_error("NoSuchBucket", f"Bucket '{bucket}' does not exist", 404)
store = _website_domains()
if not store.get_bucket(domain):
return _json_error("NotFound", f"No mapping found for domain '{domain}'", 404)
store.set_mapping(domain, bucket)
logger.info("Website domain mapping updated: %s -> %s", domain, bucket)
return jsonify({"domain": domain, "bucket": bucket})
@admin_api_bp.route("/website-domains/<domain>", methods=["DELETE"])
@limiter.limit(lambda: _get_admin_rate_limit())
def delete_website_domain(domain: str):
principal, error = _require_admin()
if error:
return error
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
return _json_error("InvalidRequest", "Website hosting is not enabled", 400)
domain = normalize_domain(domain)
if not _website_domains().delete_mapping(domain):
return _json_error("NotFound", f"No mapping found for domain '{domain}'", 404)
logger.info("Website domain mapping deleted: %s", domain)
return Response(status=204)

View File

@@ -149,6 +149,7 @@ class AppConfig:
num_trusted_proxies: int num_trusted_proxies: int
allowed_redirect_hosts: list[str] allowed_redirect_hosts: list[str]
allow_internal_endpoints: bool allow_internal_endpoints: bool
website_hosting_enabled: bool
@classmethod @classmethod
def from_env(cls, overrides: Optional[Dict[str, Any]] = None) -> "AppConfig": def from_env(cls, overrides: Optional[Dict[str, Any]] = None) -> "AppConfig":
@@ -317,6 +318,7 @@ class AppConfig:
allowed_redirect_hosts_raw = _get("ALLOWED_REDIRECT_HOSTS", "") allowed_redirect_hosts_raw = _get("ALLOWED_REDIRECT_HOSTS", "")
allowed_redirect_hosts = [h.strip() for h in str(allowed_redirect_hosts_raw).split(",") if h.strip()] allowed_redirect_hosts = [h.strip() for h in str(allowed_redirect_hosts_raw).split(",") if h.strip()]
allow_internal_endpoints = str(_get("ALLOW_INTERNAL_ENDPOINTS", "0")).lower() in {"1", "true", "yes", "on"} allow_internal_endpoints = str(_get("ALLOW_INTERNAL_ENDPOINTS", "0")).lower() in {"1", "true", "yes", "on"}
website_hosting_enabled = str(_get("WEBSITE_HOSTING_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
return cls(storage_root=storage_root, return cls(storage_root=storage_root,
max_upload_size=max_upload_size, max_upload_size=max_upload_size,
@@ -403,7 +405,8 @@ class AppConfig:
ratelimit_admin=ratelimit_admin, ratelimit_admin=ratelimit_admin,
num_trusted_proxies=num_trusted_proxies, num_trusted_proxies=num_trusted_proxies,
allowed_redirect_hosts=allowed_redirect_hosts, allowed_redirect_hosts=allowed_redirect_hosts,
allow_internal_endpoints=allow_internal_endpoints) allow_internal_endpoints=allow_internal_endpoints,
website_hosting_enabled=website_hosting_enabled)
def validate_and_report(self) -> list[str]: def validate_and_report(self) -> list[str]:
"""Validate configuration and return a list of warnings/issues. """Validate configuration and return a list of warnings/issues.
@@ -509,6 +512,8 @@ class AppConfig:
print(f" ENCRYPTION: Enabled (Master key: {self.encryption_master_key_path})") print(f" ENCRYPTION: Enabled (Master key: {self.encryption_master_key_path})")
if self.kms_enabled: if self.kms_enabled:
print(f" KMS: Enabled (Keys: {self.kms_keys_path})") print(f" KMS: Enabled (Keys: {self.kms_keys_path})")
if self.website_hosting_enabled:
print(f" WEBSITE_HOSTING: Enabled")
def _auto(flag: bool) -> str: def _auto(flag: bool) -> str:
return " (auto)" if flag else "" return " (auto)" if flag else ""
print(f" SERVER_THREADS: {self.server_threads}{_auto(self.server_threads_auto)}") print(f" SERVER_THREADS: {self.server_threads}{_auto(self.server_threads_auto)}")
@@ -611,4 +616,5 @@ class AppConfig:
"NUM_TRUSTED_PROXIES": self.num_trusted_proxies, "NUM_TRUSTED_PROXIES": self.num_trusted_proxies,
"ALLOWED_REDIRECT_HOSTS": self.allowed_redirect_hosts, "ALLOWED_REDIRECT_HOSTS": self.allowed_redirect_hosts,
"ALLOW_INTERNAL_ENDPOINTS": self.allow_internal_endpoints, "ALLOW_INTERNAL_ENDPOINTS": self.allow_internal_endpoints,
"WEBSITE_HOSTING_ENABLED": self.website_hosting_enabled,
} }

View File

@@ -274,5 +274,11 @@ class EncryptedObjectStorage:
def set_bucket_quota(self, bucket_name: str, *, max_bytes=None, max_objects=None): def set_bucket_quota(self, bucket_name: str, *, max_bytes=None, max_objects=None):
return self.storage.set_bucket_quota(bucket_name, max_bytes=max_bytes, max_objects=max_objects) return self.storage.set_bucket_quota(bucket_name, max_bytes=max_bytes, max_objects=max_objects)
def get_bucket_website(self, bucket_name: str):
return self.storage.get_bucket_website(bucket_name)
def set_bucket_website(self, bucket_name: str, website_config):
return self.storage.set_bucket_website(bucket_name, website_config)
def _compute_etag(self, path: Path) -> str: def _compute_etag(self, path: Path) -> str:
return self.storage._compute_etag(path) return self.storage._compute_etag(path)

View File

@@ -2,6 +2,7 @@ from __future__ import annotations
import json import json
import logging import logging
import random
import threading import threading
import time import time
from dataclasses import dataclass, field from dataclasses import dataclass, field
@@ -9,6 +10,8 @@ from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
MAX_LATENCY_SAMPLES = 5000
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -22,6 +25,17 @@ class OperationStats:
latency_max_ms: float = 0.0 latency_max_ms: float = 0.0
bytes_in: int = 0 bytes_in: int = 0
bytes_out: int = 0 bytes_out: int = 0
latency_samples: List[float] = field(default_factory=list)
@staticmethod
def _compute_percentile(sorted_data: List[float], p: float) -> float:
if not sorted_data:
return 0.0
k = (len(sorted_data) - 1) * (p / 100.0)
f = int(k)
c = min(f + 1, len(sorted_data) - 1)
d = k - f
return sorted_data[f] + d * (sorted_data[c] - sorted_data[f])
def record(self, latency_ms: float, success: bool, bytes_in: int = 0, bytes_out: int = 0) -> None: def record(self, latency_ms: float, success: bool, bytes_in: int = 0, bytes_out: int = 0) -> None:
self.count += 1 self.count += 1
@@ -36,10 +50,17 @@ class OperationStats:
self.latency_max_ms = latency_ms self.latency_max_ms = latency_ms
self.bytes_in += bytes_in self.bytes_in += bytes_in
self.bytes_out += bytes_out self.bytes_out += bytes_out
if len(self.latency_samples) < MAX_LATENCY_SAMPLES:
self.latency_samples.append(latency_ms)
else:
j = random.randint(0, self.count - 1)
if j < MAX_LATENCY_SAMPLES:
self.latency_samples[j] = latency_ms
def to_dict(self) -> Dict[str, Any]: def to_dict(self) -> Dict[str, Any]:
avg_latency = self.latency_sum_ms / self.count if self.count > 0 else 0.0 avg_latency = self.latency_sum_ms / self.count if self.count > 0 else 0.0
min_latency = self.latency_min_ms if self.latency_min_ms != float("inf") else 0.0 min_latency = self.latency_min_ms if self.latency_min_ms != float("inf") else 0.0
sorted_latencies = sorted(self.latency_samples)
return { return {
"count": self.count, "count": self.count,
"success_count": self.success_count, "success_count": self.success_count,
@@ -47,6 +68,9 @@ class OperationStats:
"latency_avg_ms": round(avg_latency, 2), "latency_avg_ms": round(avg_latency, 2),
"latency_min_ms": round(min_latency, 2), "latency_min_ms": round(min_latency, 2),
"latency_max_ms": round(self.latency_max_ms, 2), "latency_max_ms": round(self.latency_max_ms, 2),
"latency_p50_ms": round(self._compute_percentile(sorted_latencies, 50), 2),
"latency_p95_ms": round(self._compute_percentile(sorted_latencies, 95), 2),
"latency_p99_ms": round(self._compute_percentile(sorted_latencies, 99), 2),
"bytes_in": self.bytes_in, "bytes_in": self.bytes_in,
"bytes_out": self.bytes_out, "bytes_out": self.bytes_out,
} }
@@ -62,6 +86,11 @@ class OperationStats:
self.latency_max_ms = other.latency_max_ms self.latency_max_ms = other.latency_max_ms
self.bytes_in += other.bytes_in self.bytes_in += other.bytes_in
self.bytes_out += other.bytes_out self.bytes_out += other.bytes_out
combined = self.latency_samples + other.latency_samples
if len(combined) > MAX_LATENCY_SAMPLES:
random.shuffle(combined)
combined = combined[:MAX_LATENCY_SAMPLES]
self.latency_samples = combined
@dataclass @dataclass

View File

@@ -17,6 +17,13 @@ from urllib.parse import quote, urlencode, urlparse, unquote
from xml.etree.ElementTree import Element, SubElement, tostring, ParseError from xml.etree.ElementTree import Element, SubElement, tostring, ParseError
from defusedxml.ElementTree import fromstring from defusedxml.ElementTree import fromstring
try:
import myfsio_core as _rc
_HAS_RUST = True
except ImportError:
_rc = None
_HAS_RUST = False
from flask import Blueprint, Response, current_app, jsonify, request, g from flask import Blueprint, Response, current_app, jsonify, request, g
from werkzeug.http import http_date from werkzeug.http import http_date
@@ -192,11 +199,16 @@ _SIGNING_KEY_CACHE_MAX_SIZE = 256
def clear_signing_key_cache() -> None: def clear_signing_key_cache() -> None:
if _HAS_RUST:
_rc.clear_signing_key_cache()
with _SIGNING_KEY_CACHE_LOCK: with _SIGNING_KEY_CACHE_LOCK:
_SIGNING_KEY_CACHE.clear() _SIGNING_KEY_CACHE.clear()
def _get_signature_key(key: str, date_stamp: str, region_name: str, service_name: str) -> bytes: def _get_signature_key(key: str, date_stamp: str, region_name: str, service_name: str) -> bytes:
if _HAS_RUST:
return bytes(_rc.derive_signing_key(key, date_stamp, region_name, service_name))
cache_key = (key, date_stamp, region_name, service_name) cache_key = (key, date_stamp, region_name, service_name)
now = time.time() now = time.time()
@@ -314,9 +326,13 @@ def _verify_sigv4_header(req: Any, auth_header: str) -> Principal | None:
raise IamError("Required headers not signed") raise IamError("Required headers not signed")
credential_scope = f"{date_stamp}/{region}/{service}/aws4_request" credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}"
signing_key = _get_signature_key(secret_key, date_stamp, region, service) signing_key = _get_signature_key(secret_key, date_stamp, region, service)
calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() if _HAS_RUST:
string_to_sign = _rc.build_string_to_sign(amz_date, credential_scope, canonical_request)
calculated_signature = _rc.compute_signature(signing_key, string_to_sign)
else:
string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}"
calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
if not hmac.compare_digest(calculated_signature, signature): if not hmac.compare_digest(calculated_signature, signature):
if current_app.config.get("DEBUG_SIGV4"): if current_app.config.get("DEBUG_SIGV4"):
@@ -400,18 +416,15 @@ def _verify_sigv4_query(req: Any) -> Principal | None:
payload_hash payload_hash
]) ])
algorithm = "AWS4-HMAC-SHA256"
credential_scope = f"{date_stamp}/{region}/{service}/aws4_request" credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
hashed_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
string_to_sign = "\n".join([
algorithm,
amz_date,
credential_scope,
hashed_request
])
signing_key = _get_signature_key(secret_key, date_stamp, region, service) signing_key = _get_signature_key(secret_key, date_stamp, region, service)
calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() if _HAS_RUST:
string_to_sign = _rc.build_string_to_sign(amz_date, credential_scope, canonical_request)
calculated_signature = _rc.compute_signature(signing_key, string_to_sign)
else:
hashed_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashed_request}"
calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
if not hmac.compare_digest(calculated_signature, signature): if not hmac.compare_digest(calculated_signature, signature):
raise IamError("SignatureDoesNotMatch") raise IamError("SignatureDoesNotMatch")
@@ -1027,6 +1040,7 @@ def _maybe_handle_bucket_subresource(bucket_name: str) -> Response | None:
"uploads": _bucket_uploads_handler, "uploads": _bucket_uploads_handler,
"policy": _bucket_policy_handler, "policy": _bucket_policy_handler,
"replication": _bucket_replication_handler, "replication": _bucket_replication_handler,
"website": _bucket_website_handler,
} }
requested = [key for key in handlers if key in request.args] requested = [key for key in handlers if key in request.args]
if not requested: if not requested:
@@ -3060,6 +3074,79 @@ def _parse_replication_config(bucket_name: str, payload: bytes):
) )
def _bucket_website_handler(bucket_name: str) -> Response:
if request.method not in {"GET", "PUT", "DELETE"}:
return _method_not_allowed(["GET", "PUT", "DELETE"])
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
return _error_response("InvalidRequest", "Website hosting is not enabled", 400)
principal, error = _require_principal()
if error:
return error
try:
_authorize_action(principal, bucket_name, "policy")
except IamError as exc:
return _error_response("AccessDenied", str(exc), 403)
storage = _storage()
if request.method == "GET":
try:
config = storage.get_bucket_website(bucket_name)
except StorageError as exc:
return _error_response("NoSuchBucket", str(exc), 404)
if not config:
return _error_response("NoSuchWebsiteConfiguration", "The specified bucket does not have a website configuration", 404)
root = Element("WebsiteConfiguration")
root.set("xmlns", S3_NS)
index_doc = config.get("index_document")
if index_doc:
idx_el = SubElement(root, "IndexDocument")
SubElement(idx_el, "Suffix").text = index_doc
error_doc = config.get("error_document")
if error_doc:
err_el = SubElement(root, "ErrorDocument")
SubElement(err_el, "Key").text = error_doc
return _xml_response(root)
if request.method == "DELETE":
try:
storage.set_bucket_website(bucket_name, None)
except StorageError as exc:
return _error_response("NoSuchBucket", str(exc), 404)
current_app.logger.info("Bucket website config deleted", extra={"bucket": bucket_name})
return Response(status=204)
ct_error = _require_xml_content_type()
if ct_error:
return ct_error
payload = request.get_data(cache=False) or b""
if not payload.strip():
return _error_response("MalformedXML", "Request body is required", 400)
try:
root = _parse_xml_with_limit(payload)
except ParseError:
return _error_response("MalformedXML", "Unable to parse XML document", 400)
if _strip_ns(root.tag) != "WebsiteConfiguration":
return _error_response("MalformedXML", "Root element must be WebsiteConfiguration", 400)
index_el = _find_element(root, "IndexDocument")
if index_el is None:
return _error_response("InvalidArgument", "IndexDocument is required", 400)
suffix_el = _find_element(index_el, "Suffix")
if suffix_el is None or not (suffix_el.text or "").strip():
return _error_response("InvalidArgument", "IndexDocument Suffix is required", 400)
index_suffix = suffix_el.text.strip()
if "/" in index_suffix:
return _error_response("InvalidArgument", "IndexDocument Suffix must not contain '/'", 400)
website_config: Dict[str, Any] = {"index_document": index_suffix}
error_el = _find_element(root, "ErrorDocument")
if error_el is not None:
key_el = _find_element(error_el, "Key")
if key_el is not None and (key_el.text or "").strip():
website_config["error_document"] = key_el.text.strip()
try:
storage.set_bucket_website(bucket_name, website_config)
except StorageError as exc:
return _error_response("NoSuchBucket", str(exc), 404)
current_app.logger.info("Bucket website config updated", extra={"bucket": bucket_name, "index": index_suffix})
return Response(status=200)
def _parse_destination_arn(arn: str) -> tuple: def _parse_destination_arn(arn: str) -> tuple:
if not arn.startswith("arn:aws:s3:::"): if not arn.startswith("arn:aws:s3:::"):
raise ValueError(f"Invalid ARN format: {arn}") raise ValueError(f"Invalid ARN format: {arn}")

View File

@@ -18,6 +18,13 @@ from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from typing import Any, BinaryIO, Dict, Generator, List, Optional from typing import Any, BinaryIO, Dict, Generator, List, Optional
try:
import myfsio_core as _rc
_HAS_RUST = True
except ImportError:
_rc = None
_HAS_RUST = False
# Platform-specific file locking # Platform-specific file locking
if os.name == "nt": if os.name == "nt":
import msvcrt import msvcrt
@@ -220,6 +227,11 @@ class ObjectStorage:
raise BucketNotFoundError("Bucket does not exist") raise BucketNotFoundError("Bucket does not exist")
def _validate_bucket_name(self, bucket_name: str) -> None: def _validate_bucket_name(self, bucket_name: str) -> None:
if _HAS_RUST:
error = _rc.validate_bucket_name(bucket_name)
if error:
raise StorageError(error)
return
if len(bucket_name) < 3 or len(bucket_name) > 63: if len(bucket_name) < 3 or len(bucket_name) > 63:
raise StorageError("Bucket name must be between 3 and 63 characters") raise StorageError("Bucket name must be between 3 and 63 characters")
if not re.match(r"^[a-z0-9][a-z0-9.-]*[a-z0-9]$", bucket_name): if not re.match(r"^[a-z0-9][a-z0-9.-]*[a-z0-9]$", bucket_name):
@@ -688,10 +700,19 @@ class ObjectStorage:
return lifecycle if isinstance(lifecycle, list) else None return lifecycle if isinstance(lifecycle, list) else None
def set_bucket_lifecycle(self, bucket_name: str, rules: Optional[List[Dict[str, Any]]]) -> None: def set_bucket_lifecycle(self, bucket_name: str, rules: Optional[List[Dict[str, Any]]]) -> None:
"""Set lifecycle configuration for bucket."""
bucket_path = self._require_bucket_path(bucket_name) bucket_path = self._require_bucket_path(bucket_name)
self._set_bucket_config_entry(bucket_path.name, "lifecycle", rules) self._set_bucket_config_entry(bucket_path.name, "lifecycle", rules)
def get_bucket_website(self, bucket_name: str) -> Optional[Dict[str, Any]]:
bucket_path = self._require_bucket_path(bucket_name)
config = self._read_bucket_config(bucket_path.name)
website = config.get("website")
return website if isinstance(website, dict) else None
def set_bucket_website(self, bucket_name: str, website_config: Optional[Dict[str, Any]]) -> None:
bucket_path = self._require_bucket_path(bucket_name)
self._set_bucket_config_entry(bucket_path.name, "website", website_config)
def get_bucket_quota(self, bucket_name: str) -> Dict[str, Any]: def get_bucket_quota(self, bucket_name: str) -> Dict[str, Any]:
"""Get quota configuration for bucket. """Get quota configuration for bucket.
@@ -2124,6 +2145,18 @@ class ObjectStorage:
@staticmethod @staticmethod
def _sanitize_object_key(object_key: str, max_length_bytes: int = 1024) -> Path: def _sanitize_object_key(object_key: str, max_length_bytes: int = 1024) -> Path:
if _HAS_RUST:
error = _rc.validate_object_key(object_key, max_length_bytes, os.name == "nt")
if error:
raise StorageError(error)
normalized = unicodedata.normalize("NFC", object_key)
candidate = Path(normalized)
if candidate.is_absolute():
raise StorageError("Absolute object keys are not allowed")
if getattr(candidate, "drive", ""):
raise StorageError("Object key cannot include a drive letter")
return Path(*candidate.parts) if candidate.parts else candidate
if not object_key: if not object_key:
raise StorageError("Object key required") raise StorageError("Object key required")
if "\x00" in object_key: if "\x00" in object_key:
@@ -2165,6 +2198,8 @@ class ObjectStorage:
@staticmethod @staticmethod
def _compute_etag(path: Path) -> str: def _compute_etag(path: Path) -> str:
if _HAS_RUST:
return _rc.md5_file(str(path))
checksum = hashlib.md5() checksum = hashlib.md5()
with path.open("rb") as handle: with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(8192), b""): for chunk in iter(lambda: handle.read(8192), b""):

240
app/ui.py
View File

@@ -51,6 +51,7 @@ from .s3_client import (
from .secret_store import EphemeralSecretStore from .secret_store import EphemeralSecretStore
from .site_registry import SiteRegistry, SiteInfo, PeerSite from .site_registry import SiteRegistry, SiteInfo, PeerSite
from .storage import ObjectStorage, StorageError from .storage import ObjectStorage, StorageError
from .website_domains import normalize_domain, is_valid_domain
ui_bp = Blueprint("ui", __name__, template_folder="../templates", url_prefix="/ui") ui_bp = Blueprint("ui", __name__, template_folder="../templates", url_prefix="/ui")
@@ -287,6 +288,7 @@ def inject_nav_state() -> dict[str, Any]:
"principal": principal, "principal": principal,
"can_manage_iam": can_manage, "can_manage_iam": can_manage,
"can_view_metrics": can_manage, "can_view_metrics": can_manage,
"website_hosting_nav": can_manage and current_app.config.get("WEBSITE_HOSTING_ENABLED", False),
"csrf_token": generate_csrf, "csrf_token": generate_csrf,
} }
@@ -498,12 +500,20 @@ def bucket_detail(bucket_name: str):
encryption_enabled = current_app.config.get("ENCRYPTION_ENABLED", False) encryption_enabled = current_app.config.get("ENCRYPTION_ENABLED", False)
lifecycle_enabled = current_app.config.get("LIFECYCLE_ENABLED", False) lifecycle_enabled = current_app.config.get("LIFECYCLE_ENABLED", False)
site_sync_enabled = current_app.config.get("SITE_SYNC_ENABLED", False) site_sync_enabled = current_app.config.get("SITE_SYNC_ENABLED", False)
website_hosting_enabled = current_app.config.get("WEBSITE_HOSTING_ENABLED", False)
can_manage_encryption = can_manage_versioning can_manage_encryption = can_manage_versioning
bucket_quota = storage.get_bucket_quota(bucket_name) bucket_quota = storage.get_bucket_quota(bucket_name)
bucket_stats = storage.bucket_stats(bucket_name) bucket_stats = storage.bucket_stats(bucket_name)
can_manage_quota = is_replication_admin can_manage_quota = is_replication_admin
website_config = None
if website_hosting_enabled:
try:
website_config = storage.get_bucket_website(bucket_name)
except StorageError:
website_config = None
objects_api_url = url_for("ui.list_bucket_objects", bucket_name=bucket_name) objects_api_url = url_for("ui.list_bucket_objects", bucket_name=bucket_name)
objects_stream_url = url_for("ui.stream_bucket_objects", bucket_name=bucket_name) objects_stream_url = url_for("ui.stream_bucket_objects", bucket_name=bucket_name)
@@ -546,6 +556,9 @@ def bucket_detail(bucket_name: str):
bucket_stats=bucket_stats, bucket_stats=bucket_stats,
can_manage_quota=can_manage_quota, can_manage_quota=can_manage_quota,
site_sync_enabled=site_sync_enabled, site_sync_enabled=site_sync_enabled,
website_hosting_enabled=website_hosting_enabled,
website_config=website_config,
can_manage_website=can_edit_policy,
) )
@@ -1610,6 +1623,75 @@ def update_bucket_encryption(bucket_name: str):
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties")) return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
@ui_bp.post("/buckets/<bucket_name>/website")
def update_bucket_website(bucket_name: str):
principal = _current_principal()
try:
_authorize_ui(principal, bucket_name, "policy")
except IamError as exc:
if _wants_json():
return jsonify({"error": _friendly_error_message(exc)}), 403
flash(_friendly_error_message(exc), "danger")
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
if _wants_json():
return jsonify({"error": "Website hosting is not enabled"}), 400
flash("Website hosting is not enabled", "danger")
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
action = request.form.get("action", "enable")
if action == "disable":
try:
_storage().set_bucket_website(bucket_name, None)
if _wants_json():
return jsonify({"success": True, "message": "Static website hosting disabled", "enabled": False})
flash("Static website hosting disabled", "info")
except StorageError as exc:
if _wants_json():
return jsonify({"error": _friendly_error_message(exc)}), 400
flash(_friendly_error_message(exc), "danger")
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
index_document = request.form.get("index_document", "").strip()
error_document = request.form.get("error_document", "").strip()
if not index_document:
if _wants_json():
return jsonify({"error": "Index document is required"}), 400
flash("Index document is required", "danger")
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
if "/" in index_document:
if _wants_json():
return jsonify({"error": "Index document must not contain '/'"}), 400
flash("Index document must not contain '/'", "danger")
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
website_cfg: dict[str, Any] = {"index_document": index_document}
if error_document:
website_cfg["error_document"] = error_document
try:
_storage().set_bucket_website(bucket_name, website_cfg)
if _wants_json():
return jsonify({
"success": True,
"message": "Static website hosting enabled",
"enabled": True,
"index_document": index_document,
"error_document": error_document,
})
flash("Static website hosting enabled", "success")
except StorageError as exc:
if _wants_json():
return jsonify({"error": _friendly_error_message(exc)}), 400
flash(_friendly_error_message(exc), "danger")
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
@ui_bp.get("/iam") @ui_bp.get("/iam")
def iam_dashboard(): def iam_dashboard():
principal = _current_principal() principal = _current_principal()
@@ -1961,16 +2043,17 @@ def update_connection(connection_id: str):
secret_key = request.form.get("secret_key", "").strip() secret_key = request.form.get("secret_key", "").strip()
region = request.form.get("region", "us-east-1").strip() region = request.form.get("region", "us-east-1").strip()
if not all([name, endpoint, access_key, secret_key]): if not all([name, endpoint, access_key]):
if _wants_json(): if _wants_json():
return jsonify({"error": "All fields are required"}), 400 return jsonify({"error": "Name, endpoint, and access key are required"}), 400
flash("All fields are required", "danger") flash("Name, endpoint, and access key are required", "danger")
return redirect(url_for("ui.connections_dashboard")) return redirect(url_for("ui.connections_dashboard"))
conn.name = name conn.name = name
conn.endpoint_url = endpoint conn.endpoint_url = endpoint
conn.access_key = access_key conn.access_key = access_key
conn.secret_key = secret_key if secret_key:
conn.secret_key = secret_key
conn.region = region conn.region = region
_connections().save() _connections().save()
@@ -2275,6 +2358,155 @@ def connections_dashboard():
return render_template("connections.html", connections=connections, principal=principal) return render_template("connections.html", connections=connections, principal=principal)
@ui_bp.get("/website-domains")
def website_domains_dashboard():
principal = _current_principal()
try:
_iam().authorize(principal, None, "iam:list_users")
except IamError:
flash("Access denied", "danger")
return redirect(url_for("ui.buckets_overview"))
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
flash("Website hosting is not enabled", "warning")
return redirect(url_for("ui.buckets_overview"))
store = current_app.extensions.get("website_domains")
mappings = store.list_all() if store else []
storage = _storage()
buckets = [b.name for b in storage.list_buckets()]
return render_template(
"website_domains.html",
mappings=mappings,
buckets=buckets,
principal=principal,
can_manage_iam=True,
)
@ui_bp.post("/website-domains/create")
def create_website_domain():
principal = _current_principal()
try:
_iam().authorize(principal, None, "iam:list_users")
except IamError:
if _wants_json():
return jsonify({"error": "Access denied"}), 403
flash("Access denied", "danger")
return redirect(url_for("ui.website_domains_dashboard"))
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
if _wants_json():
return jsonify({"error": "Website hosting is not enabled"}), 400
flash("Website hosting is not enabled", "warning")
return redirect(url_for("ui.buckets_overview"))
domain = normalize_domain(request.form.get("domain") or "")
bucket = (request.form.get("bucket") or "").strip()
if not domain:
if _wants_json():
return jsonify({"error": "Domain is required"}), 400
flash("Domain is required", "danger")
return redirect(url_for("ui.website_domains_dashboard"))
if not is_valid_domain(domain):
if _wants_json():
return jsonify({"error": f"Invalid domain format: '{domain}'"}), 400
flash(f"Invalid domain format: '{domain}'. Use a hostname like www.example.com", "danger")
return redirect(url_for("ui.website_domains_dashboard"))
if not bucket:
if _wants_json():
return jsonify({"error": "Bucket is required"}), 400
flash("Bucket is required", "danger")
return redirect(url_for("ui.website_domains_dashboard"))
storage = _storage()
if not storage.bucket_exists(bucket):
if _wants_json():
return jsonify({"error": f"Bucket '{bucket}' does not exist"}), 404
flash(f"Bucket '{bucket}' does not exist", "danger")
return redirect(url_for("ui.website_domains_dashboard"))
store = current_app.extensions.get("website_domains")
if store.get_bucket(domain):
if _wants_json():
return jsonify({"error": f"Domain '{domain}' is already mapped"}), 409
flash(f"Domain '{domain}' is already mapped", "danger")
return redirect(url_for("ui.website_domains_dashboard"))
store.set_mapping(domain, bucket)
if _wants_json():
return jsonify({"success": True, "domain": domain, "bucket": bucket}), 201
flash(f"Domain '{domain}' mapped to bucket '{bucket}'", "success")
return redirect(url_for("ui.website_domains_dashboard"))
@ui_bp.post("/website-domains/<domain>/update")
def update_website_domain(domain: str):
principal = _current_principal()
try:
_iam().authorize(principal, None, "iam:list_users")
except IamError:
if _wants_json():
return jsonify({"error": "Access denied"}), 403
flash("Access denied", "danger")
return redirect(url_for("ui.website_domains_dashboard"))
domain = normalize_domain(domain)
bucket = (request.form.get("bucket") or "").strip()
if not bucket:
if _wants_json():
return jsonify({"error": "Bucket is required"}), 400
flash("Bucket is required", "danger")
return redirect(url_for("ui.website_domains_dashboard"))
storage = _storage()
if not storage.bucket_exists(bucket):
if _wants_json():
return jsonify({"error": f"Bucket '{bucket}' does not exist"}), 404
flash(f"Bucket '{bucket}' does not exist", "danger")
return redirect(url_for("ui.website_domains_dashboard"))
store = current_app.extensions.get("website_domains")
if not store.get_bucket(domain):
if _wants_json():
return jsonify({"error": f"No mapping for domain '{domain}'"}), 404
flash(f"No mapping for domain '{domain}'", "danger")
return redirect(url_for("ui.website_domains_dashboard"))
store.set_mapping(domain, bucket)
if _wants_json():
return jsonify({"success": True, "domain": domain, "bucket": bucket})
flash(f"Domain '{domain}' updated to bucket '{bucket}'", "success")
return redirect(url_for("ui.website_domains_dashboard"))
@ui_bp.post("/website-domains/<domain>/delete")
def delete_website_domain(domain: str):
principal = _current_principal()
try:
_iam().authorize(principal, None, "iam:list_users")
except IamError:
if _wants_json():
return jsonify({"error": "Access denied"}), 403
flash("Access denied", "danger")
return redirect(url_for("ui.website_domains_dashboard"))
domain = normalize_domain(domain)
store = current_app.extensions.get("website_domains")
if not store.delete_mapping(domain):
if _wants_json():
return jsonify({"error": f"No mapping for domain '{domain}'"}), 404
flash(f"No mapping for domain '{domain}'", "danger")
return redirect(url_for("ui.website_domains_dashboard"))
if _wants_json():
return jsonify({"success": True})
flash(f"Domain '{domain}' mapping deleted", "success")
return redirect(url_for("ui.website_domains_dashboard"))
@ui_bp.get("/metrics") @ui_bp.get("/metrics")
def metrics_dashboard(): def metrics_dashboard():
principal = _current_principal() principal = _current_principal()

View File

@@ -1,6 +1,6 @@
from __future__ import annotations from __future__ import annotations
APP_VERSION = "0.2.8" APP_VERSION = "0.3.0"
def get_version() -> str: def get_version() -> str:

79
app/website_domains.py Normal file
View File

@@ -0,0 +1,79 @@
from __future__ import annotations
import json
import re
import threading
from pathlib import Path
from typing import Dict, List, Optional
_DOMAIN_RE = re.compile(
r"^(?!-)[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$"
)
def normalize_domain(raw: str) -> str:
raw = raw.strip().lower()
for prefix in ("https://", "http://"):
if raw.startswith(prefix):
raw = raw[len(prefix):]
raw = raw.split("/", 1)[0]
raw = raw.split("?", 1)[0]
raw = raw.split("#", 1)[0]
if ":" in raw:
raw = raw.rsplit(":", 1)[0]
return raw
def is_valid_domain(domain: str) -> bool:
if not domain or len(domain) > 253:
return False
return bool(_DOMAIN_RE.match(domain))
class WebsiteDomainStore:
def __init__(self, config_path: Path) -> None:
self.config_path = config_path
self._lock = threading.Lock()
self._domains: Dict[str, str] = {}
self.reload()
def reload(self) -> None:
if not self.config_path.exists():
self._domains = {}
return
try:
with open(self.config_path, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, dict):
self._domains = {k.lower(): v for k, v in data.items()}
else:
self._domains = {}
except (OSError, json.JSONDecodeError):
self._domains = {}
def _save(self) -> None:
self.config_path.parent.mkdir(parents=True, exist_ok=True)
with open(self.config_path, "w", encoding="utf-8") as f:
json.dump(self._domains, f, indent=2)
def list_all(self) -> List[Dict[str, str]]:
with self._lock:
return [{"domain": d, "bucket": b} for d, b in self._domains.items()]
def get_bucket(self, domain: str) -> Optional[str]:
with self._lock:
return self._domains.get(domain.lower())
def set_mapping(self, domain: str, bucket: str) -> None:
with self._lock:
self._domains[domain.lower()] = bucket
self._save()
def delete_mapping(self, domain: str) -> bool:
with self._lock:
key = domain.lower()
if key not in self._domains:
return False
del self._domains[key]
self._save()
return True

118
docs.md
View File

@@ -1552,6 +1552,9 @@ GET /<bucket>?notification # Get event notifications
PUT /<bucket>?notification # Set event notifications (webhooks) PUT /<bucket>?notification # Set event notifications (webhooks)
GET /<bucket>?object-lock # Get object lock configuration GET /<bucket>?object-lock # Get object lock configuration
PUT /<bucket>?object-lock # Set object lock configuration PUT /<bucket>?object-lock # Set object lock configuration
GET /<bucket>?website # Get website configuration
PUT /<bucket>?website # Set website configuration
DELETE /<bucket>?website # Delete website configuration
GET /<bucket>?uploads # List active multipart uploads GET /<bucket>?uploads # List active multipart uploads
GET /<bucket>?versions # List object versions GET /<bucket>?versions # List object versions
GET /<bucket>?location # Get bucket location/region GET /<bucket>?location # Get bucket location/region
@@ -1596,6 +1599,11 @@ PUT /admin/sites/<site_id> # Update peer site
DELETE /admin/sites/<site_id> # Unregister peer site DELETE /admin/sites/<site_id> # Unregister peer site
GET /admin/sites/<site_id>/health # Check peer health GET /admin/sites/<site_id>/health # Check peer health
GET /admin/topology # Get cluster topology GET /admin/topology # Get cluster topology
GET /admin/website-domains # List domain mappings
POST /admin/website-domains # Create domain mapping
GET /admin/website-domains/<domain> # Get domain mapping
PUT /admin/website-domains/<domain> # Update domain mapping
DELETE /admin/website-domains/<domain> # Delete domain mapping
# KMS API # KMS API
GET /kms/keys # List KMS keys GET /kms/keys # List KMS keys
@@ -2229,3 +2237,113 @@ curl "http://localhost:5000/my-bucket?list-type=2&start-after=photos/2024/" \
| `start-after` | Start listing after this key | | `start-after` | Start listing after this key |
| `fetch-owner` | Include owner info in response | | `fetch-owner` | Include owner info in response |
| `encoding-type` | Set to `url` for URL-encoded keys | `encoding-type` | Set to `url` for URL-encoded keys
## 26. Static Website Hosting
MyFSIO can serve S3 buckets as static websites via custom domain mappings. When a request arrives with a `Host` header matching a mapped domain, MyFSIO resolves the bucket and serves objects directly.
### Enabling
Set the environment variable:
```bash
WEBSITE_HOSTING_ENABLED=true
```
When disabled, all website hosting endpoints return 400 and domain-based serving is skipped.
### Configuration
| Variable | Default | Description |
|----------|---------|-------------|
| `WEBSITE_HOSTING_ENABLED` | `false` | Master switch for website hosting |
### Setting Up a Website
**Step 1: Configure the bucket website settings**
```bash
curl -X PUT "http://localhost:5000/my-site?website" \
-H "Authorization: ..." \
-d '<?xml version="1.0" encoding="UTF-8"?>
<WebsiteConfiguration>
<IndexDocument><Suffix>index.html</Suffix></IndexDocument>
<ErrorDocument><Key>404.html</Key></ErrorDocument>
</WebsiteConfiguration>'
```
- `IndexDocument` with `Suffix` is required (must not contain `/`)
- `ErrorDocument` is optional
**Step 2: Map a domain to the bucket**
```bash
curl -X POST "http://localhost:5000/admin/website-domains" \
-H "Authorization: ..." \
-H "Content-Type: application/json" \
-d '{"domain": "example.com", "bucket": "my-site"}'
```
**Step 3: Point your domain to MyFSIO**
For HTTP-only (direct access), point DNS to the MyFSIO host on port 5000.
For HTTPS (recommended), use a reverse proxy. The critical requirement is passing the original `Host` header so MyFSIO can match the domain to a bucket.
**nginx example:**
```nginx
server {
server_name example.com;
listen 443 ssl;
ssl_certificate /etc/ssl/certs/example.com.pem;
ssl_certificate_key /etc/ssl/private/example.com.key;
location / {
proxy_pass http://127.0.0.1:5000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
}
```
`proxy_set_header Host $host;` is required — without it, MyFSIO cannot match the incoming domain to a bucket. You do not need any path-based routing rules; MyFSIO handles all object resolution internally.
### How Domain Routing Works
1. A request arrives with `Host: example.com`
2. MyFSIO's `before_request` hook strips the port and looks up the domain in the `WebsiteDomainStore`
3. If a match is found, it loads the bucket's website config (index/error documents)
4. Object key resolution:
- `/` or trailing `/` → append `index_document` (e.g., `index.html`)
- `/path` → try exact match, then try `path/index_document`
- Not found → serve `error_document` with 404 status
5. If no domain match is found, the request falls through to normal S3 API / UI routing
### Domain Mapping Admin API
All endpoints require admin (`iam:*`) permissions.
| Method | Route | Body | Description |
|--------|-------|------|-------------|
| `GET` | `/admin/website-domains` | — | List all mappings |
| `POST` | `/admin/website-domains` | `{"domain": "...", "bucket": "..."}` | Create mapping |
| `GET` | `/admin/website-domains/<domain>` | — | Get single mapping |
| `PUT` | `/admin/website-domains/<domain>` | `{"bucket": "..."}` | Update mapping |
| `DELETE` | `/admin/website-domains/<domain>` | — | Delete mapping |
### Bucket Website API
| Method | Route | Description |
|--------|-------|-------------|
| `PUT` | `/<bucket>?website` | Set website config (XML body) |
| `GET` | `/<bucket>?website` | Get website config (XML response) |
| `DELETE` | `/<bucket>?website` | Remove website config |
### Web UI
- **Per-bucket config:** Bucket Details → Properties tab → "Static Website Hosting" card
- **Domain management:** Sidebar → "Domains" (visible when hosting is enabled and user is admin)

421
myfsio_core/Cargo.lock generated Normal file
View File

@@ -0,0 +1,421 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "aho-corasick"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
dependencies = [
"memchr",
]
[[package]]
name = "allocator-api2"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
[[package]]
name = "bitflags"
version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
[[package]]
name = "block-buffer"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array",
]
[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "cpufeatures"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
dependencies = [
"libc",
]
[[package]]
name = "crypto-common"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
"subtle",
]
[[package]]
name = "equivalent"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "generic-array"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "hashbrown"
version = "0.15.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash",
]
[[package]]
name = "heck"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "hmac"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
dependencies = [
"digest",
]
[[package]]
name = "libc"
version = "0.2.182"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
[[package]]
name = "lock_api"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
dependencies = [
"scopeguard",
]
[[package]]
name = "lru"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f8cc7106155f10bdf99a6f379688f543ad6596a415375b36a59a054ceda1198"
dependencies = [
"hashbrown",
]
[[package]]
name = "md-5"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
dependencies = [
"cfg-if",
"digest",
]
[[package]]
name = "memchr"
version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
[[package]]
name = "myfsio_core"
version = "0.1.0"
dependencies = [
"hex",
"hmac",
"lru",
"md-5",
"parking_lot",
"pyo3",
"regex",
"sha2",
"unicode-normalization",
]
[[package]]
name = "once_cell"
version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "parking_lot"
version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-link",
]
[[package]]
name = "portable-atomic"
version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
[[package]]
name = "proc-macro2"
version = "1.0.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
dependencies = [
"unicode-ident",
]
[[package]]
name = "pyo3"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c738662e2181be11cb82487628404254902bb3225d8e9e99c31f3ef82a405c"
dependencies = [
"libc",
"once_cell",
"portable-atomic",
"pyo3-build-config",
"pyo3-ffi",
"pyo3-macros",
]
[[package]]
name = "pyo3-build-config"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9ca0864a7dd3c133a7f3f020cbff2e12e88420da854c35540fd20ce2d60e435"
dependencies = [
"target-lexicon",
]
[[package]]
name = "pyo3-ffi"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dfc1956b709823164763a34cc42bbfd26b8730afa77809a3df8b94a3ae3b059"
dependencies = [
"libc",
"pyo3-build-config",
]
[[package]]
name = "pyo3-macros"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29dc660ad948bae134d579661d08033fbb1918f4529c3bbe3257a68f2009ddf2"
dependencies = [
"proc-macro2",
"pyo3-macros-backend",
"quote",
"syn",
]
[[package]]
name = "pyo3-macros-backend"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78cd6c6d718acfcedf26c3d21fe0f053624368b0d44298c55d7138fde9331f7"
dependencies = [
"heck",
"proc-macro2",
"pyo3-build-config",
"quote",
"syn",
]
[[package]]
name = "quote"
version = "1.0.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
dependencies = [
"proc-macro2",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
dependencies = [
"bitflags",
]
[[package]]
name = "regex"
version = "1.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c"
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "sha2"
version = "0.10.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "smallvec"
version = "1.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
[[package]]
name = "subtle"
version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "syn"
version = "2.0.116"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3df424c70518695237746f84cede799c9c58fcb37450d7b23716568cc8bc69cb"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "target-lexicon"
version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
[[package]]
name = "tinyvec"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "typenum"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "unicode-normalization"
version = "0.1.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
dependencies = [
"tinyvec",
]
[[package]]
name = "version_check"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"

19
myfsio_core/Cargo.toml Normal file
View File

@@ -0,0 +1,19 @@
[package]
name = "myfsio_core"
version = "0.1.0"
edition = "2021"
[lib]
name = "myfsio_core"
crate-type = ["cdylib"]
[dependencies]
pyo3 = { version = "0.28", features = ["extension-module"] }
hmac = "0.12"
sha2 = "0.10"
md-5 = "0.10"
hex = "0.4"
unicode-normalization = "0.1"
regex = "1"
lru = "0.14"
parking_lot = "0.12"

View File

@@ -0,0 +1,11 @@
[build-system]
requires = ["maturin>=1.0,<2.0"]
build-backend = "maturin"
[project]
name = "myfsio_core"
version = "0.1.0"
requires-python = ">=3.10"
[tool.maturin]
features = ["pyo3/extension-module"]

View File

@@ -0,0 +1,90 @@
use md5::{Digest, Md5};
use pyo3::exceptions::PyIOError;
use pyo3::prelude::*;
use sha2::Sha256;
use std::fs::File;
use std::io::Read;
const CHUNK_SIZE: usize = 65536;
#[pyfunction]
pub fn md5_file(py: Python<'_>, path: &str) -> PyResult<String> {
let path = path.to_owned();
py.detach(move || {
let mut file = File::open(&path)
.map_err(|e| PyIOError::new_err(format!("Failed to open file: {}", e)))?;
let mut hasher = Md5::new();
let mut buf = vec![0u8; CHUNK_SIZE];
loop {
let n = file
.read(&mut buf)
.map_err(|e| PyIOError::new_err(format!("Failed to read file: {}", e)))?;
if n == 0 {
break;
}
hasher.update(&buf[..n]);
}
Ok(format!("{:x}", hasher.finalize()))
})
}
#[pyfunction]
pub fn md5_bytes(data: &[u8]) -> String {
let mut hasher = Md5::new();
hasher.update(data);
format!("{:x}", hasher.finalize())
}
#[pyfunction]
pub fn sha256_file(py: Python<'_>, path: &str) -> PyResult<String> {
let path = path.to_owned();
py.detach(move || {
let mut file = File::open(&path)
.map_err(|e| PyIOError::new_err(format!("Failed to open file: {}", e)))?;
let mut hasher = Sha256::new();
let mut buf = vec![0u8; CHUNK_SIZE];
loop {
let n = file
.read(&mut buf)
.map_err(|e| PyIOError::new_err(format!("Failed to read file: {}", e)))?;
if n == 0 {
break;
}
hasher.update(&buf[..n]);
}
Ok(format!("{:x}", hasher.finalize()))
})
}
#[pyfunction]
pub fn sha256_bytes(data: &[u8]) -> String {
let mut hasher = Sha256::new();
hasher.update(data);
format!("{:x}", hasher.finalize())
}
#[pyfunction]
pub fn md5_sha256_file(py: Python<'_>, path: &str) -> PyResult<(String, String)> {
let path = path.to_owned();
py.detach(move || {
let mut file = File::open(&path)
.map_err(|e| PyIOError::new_err(format!("Failed to open file: {}", e)))?;
let mut md5_hasher = Md5::new();
let mut sha_hasher = Sha256::new();
let mut buf = vec![0u8; CHUNK_SIZE];
loop {
let n = file
.read(&mut buf)
.map_err(|e| PyIOError::new_err(format!("Failed to read file: {}", e)))?;
if n == 0 {
break;
}
md5_hasher.update(&buf[..n]);
sha_hasher.update(&buf[..n]);
}
Ok((
format!("{:x}", md5_hasher.finalize()),
format!("{:x}", sha_hasher.finalize()),
))
})
}

30
myfsio_core/src/lib.rs Normal file
View File

@@ -0,0 +1,30 @@
mod hashing;
mod sigv4;
mod validation;
use pyo3::prelude::*;
#[pymodule]
mod myfsio_core {
use super::*;
#[pymodule_init]
fn init(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_function(wrap_pyfunction!(sigv4::derive_signing_key, m)?)?;
m.add_function(wrap_pyfunction!(sigv4::compute_signature, m)?)?;
m.add_function(wrap_pyfunction!(sigv4::build_string_to_sign, m)?)?;
m.add_function(wrap_pyfunction!(sigv4::constant_time_compare, m)?)?;
m.add_function(wrap_pyfunction!(sigv4::clear_signing_key_cache, m)?)?;
m.add_function(wrap_pyfunction!(hashing::md5_file, m)?)?;
m.add_function(wrap_pyfunction!(hashing::md5_bytes, m)?)?;
m.add_function(wrap_pyfunction!(hashing::sha256_file, m)?)?;
m.add_function(wrap_pyfunction!(hashing::sha256_bytes, m)?)?;
m.add_function(wrap_pyfunction!(hashing::md5_sha256_file, m)?)?;
m.add_function(wrap_pyfunction!(validation::validate_object_key, m)?)?;
m.add_function(wrap_pyfunction!(validation::validate_bucket_name, m)?)?;
Ok(())
}
}

108
myfsio_core/src/sigv4.rs Normal file
View File

@@ -0,0 +1,108 @@
use hmac::{Hmac, Mac};
use lru::LruCache;
use parking_lot::Mutex;
use pyo3::prelude::*;
use sha2::{Digest, Sha256};
use std::num::NonZeroUsize;
use std::sync::LazyLock;
use std::time::Instant;
type HmacSha256 = Hmac<Sha256>;
struct CacheEntry {
key: Vec<u8>,
created: Instant,
}
static SIGNING_KEY_CACHE: LazyLock<Mutex<LruCache<(String, String, String, String), CacheEntry>>> =
LazyLock::new(|| Mutex::new(LruCache::new(NonZeroUsize::new(256).unwrap())));
const CACHE_TTL_SECS: u64 = 60;
fn hmac_sha256(key: &[u8], msg: &[u8]) -> Vec<u8> {
let mut mac = HmacSha256::new_from_slice(key).expect("HMAC key length is always valid");
mac.update(msg);
mac.finalize().into_bytes().to_vec()
}
#[pyfunction]
pub fn derive_signing_key(
secret_key: &str,
date_stamp: &str,
region: &str,
service: &str,
) -> Vec<u8> {
let cache_key = (
secret_key.to_owned(),
date_stamp.to_owned(),
region.to_owned(),
service.to_owned(),
);
{
let mut cache = SIGNING_KEY_CACHE.lock();
if let Some(entry) = cache.get(&cache_key) {
if entry.created.elapsed().as_secs() < CACHE_TTL_SECS {
return entry.key.clone();
}
cache.pop(&cache_key);
}
}
let k_date = hmac_sha256(format!("AWS4{}", secret_key).as_bytes(), date_stamp.as_bytes());
let k_region = hmac_sha256(&k_date, region.as_bytes());
let k_service = hmac_sha256(&k_region, service.as_bytes());
let k_signing = hmac_sha256(&k_service, b"aws4_request");
{
let mut cache = SIGNING_KEY_CACHE.lock();
cache.put(
cache_key,
CacheEntry {
key: k_signing.clone(),
created: Instant::now(),
},
);
}
k_signing
}
#[pyfunction]
pub fn compute_signature(signing_key: &[u8], string_to_sign: &str) -> String {
let sig = hmac_sha256(signing_key, string_to_sign.as_bytes());
hex::encode(sig)
}
fn sha256_hex(data: &[u8]) -> String {
let mut hasher = Sha256::new();
hasher.update(data);
hex::encode(hasher.finalize())
}
#[pyfunction]
pub fn build_string_to_sign(
amz_date: &str,
credential_scope: &str,
canonical_request: &str,
) -> String {
let cr_hash = sha256_hex(canonical_request.as_bytes());
format!("AWS4-HMAC-SHA256\n{}\n{}\n{}", amz_date, credential_scope, cr_hash)
}
#[pyfunction]
pub fn constant_time_compare(a: &str, b: &str) -> bool {
if a.len() != b.len() {
return false;
}
let mut result: u8 = 0;
for (x, y) in a.bytes().zip(b.bytes()) {
result |= x ^ y;
}
result == 0
}
#[pyfunction]
pub fn clear_signing_key_cache() {
SIGNING_KEY_CACHE.lock().clear();
}

View File

@@ -0,0 +1,149 @@
use pyo3::prelude::*;
use std::sync::LazyLock;
use unicode_normalization::UnicodeNormalization;
const WINDOWS_RESERVED: &[&str] = &[
"CON", "PRN", "AUX", "NUL", "COM0", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7",
"COM8", "COM9", "LPT0", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8",
"LPT9",
];
const WINDOWS_ILLEGAL_CHARS: &[char] = &['<', '>', ':', '"', '/', '\\', '|', '?', '*'];
const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"];
const SYSTEM_ROOT: &str = ".myfsio.sys";
static IP_REGEX: LazyLock<regex::Regex> =
LazyLock::new(|| regex::Regex::new(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$").unwrap());
#[pyfunction]
#[pyo3(signature = (object_key, max_length_bytes=1024, is_windows=false, reserved_prefixes=None))]
pub fn validate_object_key(
object_key: &str,
max_length_bytes: usize,
is_windows: bool,
reserved_prefixes: Option<Vec<String>>,
) -> PyResult<Option<String>> {
if object_key.is_empty() {
return Ok(Some("Object key required".to_string()));
}
if object_key.contains('\0') {
return Ok(Some("Object key contains null bytes".to_string()));
}
let normalized: String = object_key.nfc().collect();
if normalized.len() > max_length_bytes {
return Ok(Some(format!(
"Object key exceeds maximum length of {} bytes",
max_length_bytes
)));
}
if normalized.starts_with('/') || normalized.starts_with('\\') {
return Ok(Some("Object key cannot start with a slash".to_string()));
}
let parts: Vec<&str> = if cfg!(windows) || is_windows {
normalized.split(['/', '\\']).collect()
} else {
normalized.split('/').collect()
};
for part in &parts {
if part.is_empty() {
continue;
}
if *part == ".." {
return Ok(Some(
"Object key contains parent directory references".to_string(),
));
}
if *part == "." {
return Ok(Some("Object key contains invalid segments".to_string()));
}
if part.chars().any(|c| (c as u32) < 32) {
return Ok(Some(
"Object key contains control characters".to_string(),
));
}
if is_windows {
if part.chars().any(|c| WINDOWS_ILLEGAL_CHARS.contains(&c)) {
return Ok(Some(
"Object key contains characters not supported on Windows filesystems"
.to_string(),
));
}
if part.ends_with(' ') || part.ends_with('.') {
return Ok(Some(
"Object key segments cannot end with spaces or periods on Windows".to_string(),
));
}
let trimmed = part.trim_end_matches(['.', ' ']).to_uppercase();
if WINDOWS_RESERVED.contains(&trimmed.as_str()) {
return Ok(Some(format!("Invalid filename segment: {}", part)));
}
}
}
let non_empty_parts: Vec<&str> = parts.iter().filter(|p| !p.is_empty()).copied().collect();
if let Some(top) = non_empty_parts.first() {
if INTERNAL_FOLDERS.contains(top) || *top == SYSTEM_ROOT {
return Ok(Some("Object key uses a reserved prefix".to_string()));
}
if let Some(ref prefixes) = reserved_prefixes {
for prefix in prefixes {
if *top == prefix.as_str() {
return Ok(Some("Object key uses a reserved prefix".to_string()));
}
}
}
}
Ok(None)
}
#[pyfunction]
pub fn validate_bucket_name(bucket_name: &str) -> Option<String> {
let len = bucket_name.len();
if len < 3 || len > 63 {
return Some("Bucket name must be between 3 and 63 characters".to_string());
}
let bytes = bucket_name.as_bytes();
if !bytes[0].is_ascii_lowercase() && !bytes[0].is_ascii_digit() {
return Some(
"Bucket name must start and end with a lowercase letter or digit".to_string(),
);
}
if !bytes[len - 1].is_ascii_lowercase() && !bytes[len - 1].is_ascii_digit() {
return Some(
"Bucket name must start and end with a lowercase letter or digit".to_string(),
);
}
for &b in bytes {
if !b.is_ascii_lowercase() && !b.is_ascii_digit() && b != b'.' && b != b'-' {
return Some(
"Bucket name can only contain lowercase letters, digits, dots, and hyphens"
.to_string(),
);
}
}
if bucket_name.contains("..") {
return Some("Bucket name must not contain consecutive periods".to_string());
}
if IP_REGEX.is_match(bucket_name) {
return Some("Bucket name must not be formatted as an IP address".to_string());
}
None
}

View File

@@ -0,0 +1 @@
{"rustc_fingerprint":13172970000770725120,"outputs":{"7971740275564407648":{"success":true,"status":"","code":0,"stdout":"___.exe\nlib___.rlib\n___.dll\n___.dll\n___.lib\n___.dll\nC:\\Users\\jun\\.rustup\\toolchains\\stable-x86_64-pc-windows-msvc\npacked\n___\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_abi=\"\"\ntarget_arch=\"x86_64\"\ntarget_endian=\"little\"\ntarget_env=\"msvc\"\ntarget_family=\"windows\"\ntarget_feature=\"cmpxchg16b\"\ntarget_feature=\"fxsr\"\ntarget_feature=\"sse\"\ntarget_feature=\"sse2\"\ntarget_feature=\"sse3\"\ntarget_has_atomic=\"128\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_os=\"windows\"\ntarget_pointer_width=\"64\"\ntarget_vendor=\"pc\"\nwindows\n","stderr":""},"17747080675513052775":{"success":true,"status":"","code":0,"stdout":"rustc 1.93.1 (01f6ddf75 2026-02-11)\nbinary: rustc\ncommit-hash: 01f6ddf7588f42ae2d7eb0a2f21d44e8e96674cf\ncommit-date: 2026-02-11\nhost: x86_64-pc-windows-msvc\nrelease: 1.93.1\nLLVM version: 21.1.8\n","stderr":""}},"successes":{}}

View File

@@ -0,0 +1,3 @@
Signature: 8a477f597d28d172789f06886806bc55
# This file is a cache directory tag created by cargo.
# For information about cache directory tags see https://bford.info/cachedir/

View File

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
801af22cf202da8e

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[\"perf-literal\", \"std\"]","declared_features":"[\"default\", \"logging\", \"perf-literal\", \"std\"]","target":7534583537114156500,"profile":2040997289075261528,"path":6364296192483896971,"deps":[[1363051979936526615,"memchr",false,11090220145123168660]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\aho-corasick-45694771b543be75\\dep-lib-aho_corasick","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[\"alloc\"]","declared_features":"[\"alloc\", \"default\", \"fresh-rust\", \"nightly\", \"serde\", \"std\"]","target":5388200169723499962,"profile":4067574213046180398,"path":10654049299693593327,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\allocator-api2-db7934dbe96de5b4\\dep-lib-allocator_api2","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
d28af275d001c358

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":6962977057026645649,"profile":1369601567987815722,"path":9853093265219907461,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\autocfg-1c4fb7a37cc3df69\\dep-lib-autocfg","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
1fbf4ba9542edced

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":4098124618827574291,"profile":2040997289075261528,"path":3658007358608479489,"deps":[[10520923840501062997,"generic_array",false,11555283918993371487]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\block-buffer-95b0ac364bec72f9\\dep-lib-block_buffer","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
37923e6f5f9687ab

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[\"core\", \"rustc-dep-of-std\"]","target":13840298032947503755,"profile":2040997289075261528,"path":4093486168504982869,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\cfg-if-be2711f84a777e73\\dep-lib-cfg_if","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
603e28136cf5763c

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":2330704043955282025,"profile":2040997289075261528,"path":13200428550696548327,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\cpufeatures-980094f8735c42d1\\dep-lib-cpufeatures","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[\"std\"]","declared_features":"[\"getrandom\", \"rand_core\", \"std\"]","target":12082577455412410174,"profile":2040997289075261528,"path":14902376638882023040,"deps":[[857979250431893282,"typenum",false,7416411392359930020],[10520923840501062997,"generic_array",false,11555283918993371487]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\crypto-common-289a508abdda3048\\dep-lib-crypto_common","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
914a617b9f05c9d8

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[\"alloc\", \"block-buffer\", \"core-api\", \"default\", \"mac\", \"std\", \"subtle\"]","declared_features":"[\"alloc\", \"blobby\", \"block-buffer\", \"const-oid\", \"core-api\", \"default\", \"dev\", \"mac\", \"oid\", \"rand_core\", \"std\", \"subtle\"]","target":7510122432137863311,"profile":2040997289075261528,"path":11503432597517024930,"deps":[[6039282458970808711,"crypto_common",false,11252724541433210505],[10626340395483396037,"block_buffer",false,17139625223017709343],[17003143334332120809,"subtle",false,8597342066671925934]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\digest-a91458bfa5613332\\dep-lib-digest","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
3b95cf48bbd7dc53

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":1524667692659508025,"profile":2040997289075261528,"path":17534356223679657546,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\equivalent-943ac856871c0988\\dep-lib-equivalent","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
b7ba5182ce570398

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[\"default\", \"std\"]","target":18077926938045032029,"profile":2040997289075261528,"path":9869209539952544870,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\foldhash-b8a92f8c10d550f7\\dep-lib-foldhash","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[\"more_lengths\"]","declared_features":"[\"more_lengths\", \"serde\", \"zeroize\"]","target":12318548087768197662,"profile":1369601567987815722,"path":13853454403963664247,"deps":[[5398981501050481332,"version_check",false,16419025953046340415]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\generic-array-2462daa120fe5936\\dep-build-script-build-script-build","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[\"more_lengths\"]","declared_features":"[\"more_lengths\", \"serde\", \"zeroize\"]","target":13084005262763373425,"profile":2040997289075261528,"path":12463275850883329568,"deps":[[857979250431893282,"typenum",false,7416411392359930020],[10520923840501062997,"build_script_build",false,16977603856295925732]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\generic-array-62216349963f3a3c\\dep-lib-generic_array","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"","declared_features":"","target":0,"profile":0,"path":0,"deps":[[10520923840501062997,"build_script_build",false,464306762232604144]],"local":[{"Precalculated":"0.14.7"}],"rustflags":[],"config":0,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
aec88a641c5288e3

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[\"allocator-api2\", \"default\", \"default-hasher\", \"equivalent\", \"inline-more\", \"raw-entry\"]","declared_features":"[\"alloc\", \"allocator-api2\", \"core\", \"default\", \"default-hasher\", \"equivalent\", \"inline-more\", \"nightly\", \"raw-entry\", \"rayon\", \"rustc-dep-of-std\", \"rustc-internal-api\", \"serde\"]","target":13796197676120832388,"profile":2040997289075261528,"path":12448322139402656924,"deps":[[5230392855116717286,"equivalent",false,6042941999404782907],[9150530836556604396,"allocator_api2",false,16398368410642502979],[10842263908529601448,"foldhash",false,10953695263156452023]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\hashbrown-510d641b592c306b\\dep-lib-hashbrown","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
ddc0b590ff80762b

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":17886154901722686619,"profile":1369601567987815722,"path":8608102977929876445,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\heck-b47c94fd2a7e00cb\\dep-lib-heck","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
41890ebff4143fa5

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[\"alloc\", \"default\", \"std\"]","declared_features":"[\"alloc\", \"default\", \"serde\", \"std\"]","target":4242469766639956503,"profile":2040997289075261528,"path":6793865871540733919,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\hex-253414d2260adcdf\\dep-lib-hex","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
3f45b8d062d94ba4

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[\"reset\", \"std\"]","target":12991177224612424488,"profile":2040997289075261528,"path":17893893568771568113,"deps":[[17475753849556516473,"digest",false,15621022965039188625]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\hmac-3297e61b9effb758\\dep-lib-hmac","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
9896adc8892b3fe4

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":8726396592336845528,"profile":1369601567987815722,"path":18304219166357541938,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\indoc-0c686c3f403a2566\\dep-lib-indoc","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

View File

@@ -0,0 +1 @@
3c0b7985f088ae56

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[\"default\", \"std\"]","declared_features":"[\"align\", \"const-extern-fn\", \"default\", \"extra_traits\", \"rustc-dep-of-std\", \"rustc-std-workspace-core\", \"std\", \"use_std\"]","target":17682796336736096309,"profile":7322064999780386650,"path":3108645287704295931,"deps":[[18365559012052052344,"build_script_build",false,12197584826291254217]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\libc-5f9f280eeaad3bb3\\dep-lib-libc","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
{"rustc":8323788817864214825,"features":"[\"default\", \"std\"]","declared_features":"[\"align\", \"const-extern-fn\", \"default\", \"extra_traits\", \"rustc-dep-of-std\", \"rustc-std-workspace-core\", \"std\", \"use_std\"]","target":5408242616063297496,"profile":8928907579149787682,"path":7198683120865577851,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\libc-70cd639287284bb6\\dep-build-script-build-script-build","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -0,0 +1 @@
This file has an mtime of when this was started.

Some files were not shown because too many files have changed in this diff Show More