31 Commits

Author SHA1 Message Date
0462a7b62e MyFSIO v0.3.0 Release
Reviewed-on: #22
2026-02-22 10:22:35 +00:00
9c2809c195 Backwards compatibility for Proxy trust config 2026-02-22 18:03:38 +08:00
fb32ca0a7d Harden security: fail-closed policies, presigned URL time/expiry validation, SSRF DNS pinning, lockout cap, proxy trust config 2026-02-22 17:55:40 +08:00
6ab702a818 Use cached etag in HEAD instead of re-hashing entire file 2026-02-22 16:01:46 +08:00
550e7d435c Move SigV4 canonical request construction to Rust unified verify function 2026-02-22 14:03:12 +08:00
776967e80d Add Rust index reader, metadata read cache, and 256KB stream chunks 2026-02-19 23:01:40 +08:00
082a7fbcd1 Move index JSON read to Rust for GIL-released parsing (serde_json) 2026-02-19 22:43:28 +08:00
ff287cf67b Improve Sites page UI/UX: dropdown actions, collapsible forms, AJAX submissions, Check All Health, safer selectors 2026-02-16 22:04:46 +08:00
bddf36d52d Fix domain mapping cross-process staleness, filter bucket dropdown to website-enabled only 2026-02-16 17:48:21 +08:00
cf6cec9cab Add 5 missing S3 API operations: DeleteBucketEncryption, GetObjectAcl, PutObjectAcl, GetObjectAttributes, GetBucketPolicyStatus 2026-02-16 16:41:27 +08:00
d425839e57 Remove Rust build artifacts from tracking, update .gitignore 2026-02-16 16:06:42 +08:00
52660570c1 Merge pull request 'MyFSIO v0.2.9 Release' (#21) from next into main
Reviewed-on: #21
2026-02-15 14:24:14 +00:00
35f61313e0 MyFSIO v0.2.8 Release
Reviewed-on: #20
2026-02-10 14:16:22 +00:00
c470cfb576 MyFSIO v0.2.7 Release
Reviewed-on: #19
2026-02-09 12:22:37 +00:00
jun
d96955deee MyFSIO v0.2.6 Release
Reviewed-on: #18
2026-02-05 16:18:03 +00:00
85181f0be6 Merge pull request 'MyFSIO v0.2.5 Release' (#17) from next into main
Reviewed-on: #17
2026-02-02 05:32:02 +00:00
d5ca7a8be1 Merge pull request 'MyFSIO v0.2.4 Release' (#16) from next into main
Reviewed-on: #16
2026-02-01 10:27:11 +00:00
476dc79e42 MyFSIO v0.2.3 Release
Reviewed-on: #15
2026-01-25 06:05:53 +00:00
bb6590fc5e Merge pull request 'MyFSIO v0.2.2 Release' (#14) from next into main
Reviewed-on: #14
2026-01-19 07:12:15 +00:00
899db3421b Merge pull request 'MyFSIO v0.2.1 Release' (#13) from next into main
Reviewed-on: #13
2026-01-12 08:03:29 +00:00
caf01d6ada Merge pull request 'MyFSIO v0.2.0 Release' (#12) from next into main
Reviewed-on: #12
2026-01-05 15:48:03 +00:00
bb366cb4cd Merge pull request 'MyFSIO v0.1.9 Release' (#10) from next into main
Reviewed-on: #10
2025-12-29 06:49:48 +00:00
a2745ff2ee Merge pull request 'MyFSIO v0.1.8 Release' (#9) from next into main
Reviewed-on: #9
2025-12-23 06:01:32 +00:00
28cb656d94 Merge pull request 'MyFSIO v0.1.7 Release' (#8) from next into main
Reviewed-on: #8
2025-12-22 03:10:35 +00:00
3c44152fc6 Merge pull request 'MyFSIO v0.1.6 Release' (#7) from next into main
Reviewed-on: #7
2025-12-21 06:30:21 +00:00
397515edce Merge pull request 'MyFSIO v0.1.5 Release' (#6) from next into main
Reviewed-on: #6
2025-12-13 15:41:03 +00:00
980fced7e4 Merge pull request 'MyFSIO v0.1.4 Release' (#5) from next into main
Reviewed-on: #5
2025-12-13 08:22:43 +00:00
bae5009ec4 Merge pull request 'Release v0.1.3' (#4) from next into main
Reviewed-on: #4
2025-12-03 04:14:57 +00:00
233780617f Merge pull request 'Release V0.1.2' (#3) from next into main
Reviewed-on: #3
2025-11-26 04:59:15 +00:00
fd8fb21517 Merge pull request 'Prepare for binary release' (#2) from next into main
Reviewed-on: #2
2025-11-22 12:33:38 +00:00
c6cbe822e1 Merge pull request 'Release v0.1.1' (#1) from next into main
Reviewed-on: #1
2025-11-22 12:31:27 +00:00
502 changed files with 1084 additions and 2193 deletions

4
.gitignore vendored
View File

@@ -26,6 +26,10 @@ dist/
*.egg-info/ *.egg-info/
.eggs/ .eggs/
# Rust / maturin build artifacts
myfsio_core/target/
myfsio_core/Cargo.lock
# Local runtime artifacts # Local runtime artifacts
logs/ logs/
*.log *.log

View File

@@ -3,6 +3,7 @@ from __future__ import annotations
import html as html_module import html as html_module
import logging import logging
import mimetypes import mimetypes
import os
import shutil import shutil
import sys import sys
import time import time
@@ -93,7 +94,14 @@ def create_app(
app.config.setdefault("WTF_CSRF_ENABLED", False) app.config.setdefault("WTF_CSRF_ENABLED", False)
# Trust X-Forwarded-* headers from proxies # Trust X-Forwarded-* headers from proxies
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1) num_proxies = app.config.get("NUM_TRUSTED_PROXIES", 1)
if num_proxies:
if "NUM_TRUSTED_PROXIES" not in os.environ:
logging.getLogger(__name__).warning(
"NUM_TRUSTED_PROXIES not set, defaulting to 1. "
"Set NUM_TRUSTED_PROXIES=0 if not behind a reverse proxy."
)
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=num_proxies, x_proto=num_proxies, x_host=num_proxies, x_prefix=num_proxies)
# Enable gzip compression for responses (10-20x smaller JSON payloads) # Enable gzip compression for responses (10-20x smaller JSON payloads)
if app.config.get("ENABLE_GZIP", True): if app.config.get("ENABLE_GZIP", True):

View File

@@ -75,7 +75,7 @@ def _evaluate_condition_operator(
expected_null = condition_values[0].lower() in ("true", "1", "yes") if condition_values else True expected_null = condition_values[0].lower() in ("true", "1", "yes") if condition_values else True
return is_null == expected_null return is_null == expected_null
return True return False
ACTION_ALIASES = { ACTION_ALIASES = {
"s3:listbucket": "list", "s3:listbucket": "list",

View File

@@ -314,7 +314,7 @@ class AppConfig:
site_region = str(_get("SITE_REGION", "us-east-1")) site_region = str(_get("SITE_REGION", "us-east-1"))
site_priority = int(_get("SITE_PRIORITY", 100)) site_priority = int(_get("SITE_PRIORITY", 100))
ratelimit_admin = _validate_rate_limit(str(_get("RATE_LIMIT_ADMIN", "60 per minute"))) ratelimit_admin = _validate_rate_limit(str(_get("RATE_LIMIT_ADMIN", "60 per minute")))
num_trusted_proxies = int(_get("NUM_TRUSTED_PROXIES", 0)) num_trusted_proxies = int(_get("NUM_TRUSTED_PROXIES", 1))
allowed_redirect_hosts_raw = _get("ALLOWED_REDIRECT_HOSTS", "") allowed_redirect_hosts_raw = _get("ALLOWED_REDIRECT_HOSTS", "")
allowed_redirect_hosts = [h.strip() for h in str(allowed_redirect_hosts_raw).split(",") if h.strip()] allowed_redirect_hosts = [h.strip() for h in str(allowed_redirect_hosts_raw).split(",") if h.strip()]
allow_internal_endpoints = str(_get("ALLOW_INTERNAL_ENDPOINTS", "0")).lower() in {"1", "true", "yes", "on"} allow_internal_endpoints = str(_get("ALLOW_INTERNAL_ENDPOINTS", "0")).lower() in {"1", "true", "yes", "on"}

View File

@@ -164,9 +164,14 @@ class IamService:
self._clear_failed_attempts(access_key) self._clear_failed_attempts(access_key)
return self._build_principal(access_key, record) return self._build_principal(access_key, record)
_MAX_LOCKOUT_KEYS = 10000
def _record_failed_attempt(self, access_key: str) -> None: def _record_failed_attempt(self, access_key: str) -> None:
if not access_key: if not access_key:
return return
if access_key not in self._failed_attempts and len(self._failed_attempts) >= self._MAX_LOCKOUT_KEYS:
oldest_key = min(self._failed_attempts, key=lambda k: self._failed_attempts[k][0] if self._failed_attempts[k] else datetime.min.replace(tzinfo=timezone.utc))
del self._failed_attempts[oldest_key]
attempts = self._failed_attempts.setdefault(access_key, deque()) attempts = self._failed_attempts.setdefault(access_key, deque())
self._prune_attempts(attempts) self._prune_attempts(attempts)
attempts.append(datetime.now(timezone.utc)) attempts.append(datetime.now(timezone.utc))

View File

@@ -15,29 +15,23 @@ from typing import Any, Dict, List, Optional
from urllib.parse import urlparse from urllib.parse import urlparse
import requests import requests
from urllib3.util.connection import create_connection as _urllib3_create_connection
def _is_safe_url(url: str, allow_internal: bool = False) -> bool: def _resolve_and_check_url(url: str, allow_internal: bool = False) -> Optional[str]:
"""Check if a URL is safe to make requests to (not internal/private).
Args:
url: The URL to check.
allow_internal: If True, allows internal/private IP addresses.
Use for self-hosted deployments on internal networks.
"""
try: try:
parsed = urlparse(url) parsed = urlparse(url)
hostname = parsed.hostname hostname = parsed.hostname
if not hostname: if not hostname:
return False return None
cloud_metadata_hosts = { cloud_metadata_hosts = {
"metadata.google.internal", "metadata.google.internal",
"169.254.169.254", "169.254.169.254",
} }
if hostname.lower() in cloud_metadata_hosts: if hostname.lower() in cloud_metadata_hosts:
return False return None
if allow_internal: if allow_internal:
return True return hostname
blocked_hosts = { blocked_hosts = {
"localhost", "localhost",
"127.0.0.1", "127.0.0.1",
@@ -46,17 +40,46 @@ def _is_safe_url(url: str, allow_internal: bool = False) -> bool:
"[::1]", "[::1]",
} }
if hostname.lower() in blocked_hosts: if hostname.lower() in blocked_hosts:
return False return None
try: try:
resolved_ip = socket.gethostbyname(hostname) resolved_ip = socket.gethostbyname(hostname)
ip = ipaddress.ip_address(resolved_ip) ip = ipaddress.ip_address(resolved_ip)
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved: if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
return False return None
return resolved_ip
except (socket.gaierror, ValueError): except (socket.gaierror, ValueError):
return False return None
return True
except Exception: except Exception:
return False return None
def _is_safe_url(url: str, allow_internal: bool = False) -> bool:
return _resolve_and_check_url(url, allow_internal) is not None
_dns_pin_lock = threading.Lock()
def _pinned_post(url: str, pinned_ip: str, **kwargs: Any) -> requests.Response:
parsed = urlparse(url)
hostname = parsed.hostname or ""
session = requests.Session()
original_create = _urllib3_create_connection
def _create_pinned(address: Any, *args: Any, **kw: Any) -> Any:
host, req_port = address
if host == hostname:
return original_create((pinned_ip, req_port), *args, **kw)
return original_create(address, *args, **kw)
import urllib3.util.connection as _conn_mod
with _dns_pin_lock:
_conn_mod.create_connection = _create_pinned
try:
return session.post(url, **kwargs)
finally:
_conn_mod.create_connection = original_create
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -344,16 +367,18 @@ class NotificationService:
self._queue.task_done() self._queue.task_done()
def _send_notification(self, event: NotificationEvent, destination: WebhookDestination) -> None: def _send_notification(self, event: NotificationEvent, destination: WebhookDestination) -> None:
if not _is_safe_url(destination.url, allow_internal=self._allow_internal_endpoints): resolved_ip = _resolve_and_check_url(destination.url, allow_internal=self._allow_internal_endpoints)
raise RuntimeError(f"Blocked request to cloud metadata service (SSRF protection): {destination.url}") if not resolved_ip:
raise RuntimeError(f"Blocked request (SSRF protection): {destination.url}")
payload = event.to_s3_event() payload = event.to_s3_event()
headers = {"Content-Type": "application/json", **destination.headers} headers = {"Content-Type": "application/json", **destination.headers}
last_error = None last_error = None
for attempt in range(destination.retry_count): for attempt in range(destination.retry_count):
try: try:
response = requests.post( response = _pinned_post(
destination.url, destination.url,
resolved_ip,
json=payload, json=payload,
headers=headers, headers=headers,
timeout=destination.timeout_seconds, timeout=destination.timeout_seconds,

View File

@@ -267,39 +267,6 @@ def _verify_sigv4_header(req: Any, auth_header: str) -> Principal | None:
if not secret_key: if not secret_key:
raise IamError("SignatureDoesNotMatch") raise IamError("SignatureDoesNotMatch")
method = req.method
canonical_uri = _get_canonical_uri(req)
query_args = []
for key, value in req.args.items(multi=True):
query_args.append((key, value))
query_args.sort(key=lambda x: (x[0], x[1]))
canonical_query_parts = []
for k, v in query_args:
canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}")
canonical_query_string = "&".join(canonical_query_parts)
signed_headers_list = signed_headers_str.split(";")
canonical_headers_parts = []
for header in signed_headers_list:
header_val = req.headers.get(header)
if header_val is None:
header_val = ""
if header.lower() == 'expect' and header_val == "":
header_val = "100-continue"
header_val = " ".join(header_val.split())
canonical_headers_parts.append(f"{header.lower()}:{header_val}\n")
canonical_headers = "".join(canonical_headers_parts)
payload_hash = req.headers.get("X-Amz-Content-Sha256")
if not payload_hash:
payload_hash = hashlib.sha256(req.get_data()).hexdigest()
canonical_request = f"{method}\n{canonical_uri}\n{canonical_query_string}\n{canonical_headers}\n{signed_headers_str}\n{payload_hash}"
amz_date = req.headers.get("X-Amz-Date") or req.headers.get("Date") amz_date = req.headers.get("X-Amz-Date") or req.headers.get("Date")
if not amz_date: if not amz_date:
raise IamError("Missing Date header") raise IamError("Missing Date header")
@@ -325,15 +292,52 @@ def _verify_sigv4_header(req: Any, auth_header: str) -> Principal | None:
if not required_headers.issubset(signed_headers_set): if not required_headers.issubset(signed_headers_set):
raise IamError("Required headers not signed") raise IamError("Required headers not signed")
canonical_uri = _get_canonical_uri(req)
payload_hash = req.headers.get("X-Amz-Content-Sha256")
if not payload_hash:
payload_hash = hashlib.sha256(req.get_data()).hexdigest()
if _HAS_RUST:
query_params = list(req.args.items(multi=True))
header_values = [(h, req.headers.get(h) or "") for h in signed_headers_str.split(";")]
if not _rc.verify_sigv4_signature(
req.method, canonical_uri, query_params, signed_headers_str,
header_values, payload_hash, amz_date, date_stamp, region,
service, secret_key, signature,
):
if current_app.config.get("DEBUG_SIGV4"):
logger.warning("SigV4 signature mismatch for %s %s", req.method, req.path)
raise IamError("SignatureDoesNotMatch")
else:
method = req.method
query_args = []
for key, value in req.args.items(multi=True):
query_args.append((key, value))
query_args.sort(key=lambda x: (x[0], x[1]))
canonical_query_parts = []
for k, v in query_args:
canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}")
canonical_query_string = "&".join(canonical_query_parts)
signed_headers_list = signed_headers_str.split(";")
canonical_headers_parts = []
for header in signed_headers_list:
header_val = req.headers.get(header)
if header_val is None:
header_val = ""
if header.lower() == 'expect' and header_val == "":
header_val = "100-continue"
header_val = " ".join(header_val.split())
canonical_headers_parts.append(f"{header.lower()}:{header_val}\n")
canonical_headers = "".join(canonical_headers_parts)
canonical_request = f"{method}\n{canonical_uri}\n{canonical_query_string}\n{canonical_headers}\n{signed_headers_str}\n{payload_hash}"
credential_scope = f"{date_stamp}/{region}/{service}/aws4_request" credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
signing_key = _get_signature_key(secret_key, date_stamp, region, service) signing_key = _get_signature_key(secret_key, date_stamp, region, service)
if _HAS_RUST:
string_to_sign = _rc.build_string_to_sign(amz_date, credential_scope, canonical_request)
calculated_signature = _rc.compute_signature(signing_key, string_to_sign)
else:
string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}" string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}"
calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
if not hmac.compare_digest(calculated_signature, signature): if not hmac.compare_digest(calculated_signature, signature):
if current_app.config.get("DEBUG_SIGV4"): if current_app.config.get("DEBUG_SIGV4"):
logger.warning("SigV4 signature mismatch for %s %s", method, req.path) logger.warning("SigV4 signature mismatch for %s %s", method, req.path)
@@ -368,12 +372,19 @@ def _verify_sigv4_query(req: Any) -> Principal | None:
raise IamError("Invalid Date format") raise IamError("Invalid Date format")
now = datetime.now(timezone.utc) now = datetime.now(timezone.utc)
tolerance = timedelta(seconds=current_app.config.get("SIGV4_TIMESTAMP_TOLERANCE_SECONDS", 900))
if req_time > now + tolerance:
raise IamError("Request date is too far in the future")
try: try:
expires_seconds = int(expires) expires_seconds = int(expires)
if expires_seconds <= 0: if expires_seconds <= 0:
raise IamError("Invalid Expires value: must be positive") raise IamError("Invalid Expires value: must be positive")
except ValueError: except ValueError:
raise IamError("Invalid Expires value: must be an integer") raise IamError("Invalid Expires value: must be an integer")
min_expiry = current_app.config.get("PRESIGNED_URL_MIN_EXPIRY_SECONDS", 1)
max_expiry = current_app.config.get("PRESIGNED_URL_MAX_EXPIRY_SECONDS", 604800)
if expires_seconds < min_expiry or expires_seconds > max_expiry:
raise IamError(f"Expiration must be between {min_expiry} second(s) and {max_expiry} seconds")
if now > req_time + timedelta(seconds=expires_seconds): if now > req_time + timedelta(seconds=expires_seconds):
raise IamError("Request expired") raise IamError("Request expired")
@@ -381,9 +392,19 @@ def _verify_sigv4_query(req: Any) -> Principal | None:
if not secret_key: if not secret_key:
raise IamError("Invalid access key") raise IamError("Invalid access key")
method = req.method
canonical_uri = _get_canonical_uri(req) canonical_uri = _get_canonical_uri(req)
if _HAS_RUST:
query_params = [(k, v) for k, v in req.args.items(multi=True) if k != "X-Amz-Signature"]
header_values = [(h, req.headers.get(h) or "") for h in signed_headers_str.split(";")]
if not _rc.verify_sigv4_signature(
req.method, canonical_uri, query_params, signed_headers_str,
header_values, "UNSIGNED-PAYLOAD", amz_date, date_stamp, region,
service, secret_key, signature,
):
raise IamError("SignatureDoesNotMatch")
else:
method = req.method
query_args = [] query_args = []
for key, value in req.args.items(multi=True): for key, value in req.args.items(multi=True):
if key != "X-Amz-Signature": if key != "X-Amz-Signature":
@@ -418,14 +439,9 @@ def _verify_sigv4_query(req: Any) -> Principal | None:
credential_scope = f"{date_stamp}/{region}/{service}/aws4_request" credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
signing_key = _get_signature_key(secret_key, date_stamp, region, service) signing_key = _get_signature_key(secret_key, date_stamp, region, service)
if _HAS_RUST:
string_to_sign = _rc.build_string_to_sign(amz_date, credential_scope, canonical_request)
calculated_signature = _rc.compute_signature(signing_key, string_to_sign)
else:
hashed_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest() hashed_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashed_request}" string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashed_request}"
calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
if not hmac.compare_digest(calculated_signature, signature): if not hmac.compare_digest(calculated_signature, signature):
raise IamError("SignatureDoesNotMatch") raise IamError("SignatureDoesNotMatch")
@@ -586,7 +602,11 @@ def _validate_presigned_request(action: str, bucket_name: str, object_key: str)
request_time = datetime.strptime(amz_date, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc) request_time = datetime.strptime(amz_date, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc)
except ValueError as exc: except ValueError as exc:
raise IamError("Invalid X-Amz-Date") from exc raise IamError("Invalid X-Amz-Date") from exc
if datetime.now(timezone.utc) > request_time + timedelta(seconds=expiry): now = datetime.now(timezone.utc)
tolerance = timedelta(seconds=current_app.config.get("SIGV4_TIMESTAMP_TOLERANCE_SECONDS", 900))
if request_time > now + tolerance:
raise IamError("Request date is too far in the future")
if now > request_time + timedelta(seconds=expiry):
raise IamError("Presigned URL expired") raise IamError("Presigned URL expired")
signed_headers_list = [header.strip().lower() for header in signed_headers.split(";") if header] signed_headers_list = [header.strip().lower() for header in signed_headers.split(";") if header]
@@ -986,7 +1006,7 @@ def _render_encryption_document(config: dict[str, Any]) -> Element:
return root return root
def _stream_file(path, chunk_size: int = 64 * 1024): def _stream_file(path, chunk_size: int = 256 * 1024):
with path.open("rb") as handle: with path.open("rb") as handle:
while True: while True:
chunk = handle.read(chunk_size) chunk = handle.read(chunk_size)
@@ -1039,6 +1059,7 @@ def _maybe_handle_bucket_subresource(bucket_name: str) -> Response | None:
"logging": _bucket_logging_handler, "logging": _bucket_logging_handler,
"uploads": _bucket_uploads_handler, "uploads": _bucket_uploads_handler,
"policy": _bucket_policy_handler, "policy": _bucket_policy_handler,
"policyStatus": _bucket_policy_status_handler,
"replication": _bucket_replication_handler, "replication": _bucket_replication_handler,
"website": _bucket_website_handler, "website": _bucket_website_handler,
} }
@@ -1321,8 +1342,8 @@ def _bucket_cors_handler(bucket_name: str) -> Response:
def _bucket_encryption_handler(bucket_name: str) -> Response: def _bucket_encryption_handler(bucket_name: str) -> Response:
if request.method not in {"GET", "PUT"}: if request.method not in {"GET", "PUT", "DELETE"}:
return _method_not_allowed(["GET", "PUT"]) return _method_not_allowed(["GET", "PUT", "DELETE"])
principal, error = _require_principal() principal, error = _require_principal()
if error: if error:
return error return error
@@ -1343,6 +1364,13 @@ def _bucket_encryption_handler(bucket_name: str) -> Response:
404, 404,
) )
return _xml_response(_render_encryption_document(config)) return _xml_response(_render_encryption_document(config))
if request.method == "DELETE":
try:
storage.set_bucket_encryption(bucket_name, None)
except StorageError as exc:
return _error_response("NoSuchBucket", str(exc), 404)
current_app.logger.info("Bucket encryption deleted", extra={"bucket": bucket_name})
return Response(status=204)
ct_error = _require_xml_content_type() ct_error = _require_xml_content_type()
if ct_error: if ct_error:
return ct_error return ct_error
@@ -1439,6 +1467,99 @@ def _bucket_acl_handler(bucket_name: str) -> Response:
return _xml_response(root) return _xml_response(root)
def _object_acl_handler(bucket_name: str, object_key: str) -> Response:
from .acl import create_canned_acl, GRANTEE_ALL_USERS, GRANTEE_AUTHENTICATED_USERS
if request.method not in {"GET", "PUT"}:
return _method_not_allowed(["GET", "PUT"])
storage = _storage()
try:
path = storage.get_object_path(bucket_name, object_key)
except (StorageError, FileNotFoundError):
return _error_response("NoSuchKey", "Object not found", 404)
if request.method == "PUT":
principal, error = _object_principal("write", bucket_name, object_key)
if error:
return error
owner_id = principal.access_key if principal else "anonymous"
canned_acl = request.headers.get("x-amz-acl", "private")
acl = create_canned_acl(canned_acl, owner_id)
acl_service = _acl()
metadata = storage.get_object_metadata(bucket_name, object_key)
metadata.update(acl_service.create_object_acl_metadata(acl))
safe_key = storage._sanitize_object_key(object_key, storage._object_key_max_length_bytes)
storage._write_metadata(bucket_name, safe_key, metadata)
current_app.logger.info("Object ACL set", extra={"bucket": bucket_name, "key": object_key, "acl": canned_acl})
return Response(status=200)
principal, error = _object_principal("read", bucket_name, object_key)
if error:
return error
owner_id = principal.access_key if principal else "anonymous"
acl_service = _acl()
metadata = storage.get_object_metadata(bucket_name, object_key)
acl = acl_service.get_object_acl(bucket_name, object_key, metadata)
if not acl:
acl = create_canned_acl("private", owner_id)
root = Element("AccessControlPolicy")
owner_el = SubElement(root, "Owner")
SubElement(owner_el, "ID").text = acl.owner
SubElement(owner_el, "DisplayName").text = acl.owner
acl_el = SubElement(root, "AccessControlList")
for grant in acl.grants:
grant_el = SubElement(acl_el, "Grant")
grantee = SubElement(grant_el, "Grantee")
if grant.grantee == GRANTEE_ALL_USERS:
grantee.set("{http://www.w3.org/2001/XMLSchema-instance}type", "Group")
SubElement(grantee, "URI").text = "http://acs.amazonaws.com/groups/global/AllUsers"
elif grant.grantee == GRANTEE_AUTHENTICATED_USERS:
grantee.set("{http://www.w3.org/2001/XMLSchema-instance}type", "Group")
SubElement(grantee, "URI").text = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
else:
grantee.set("{http://www.w3.org/2001/XMLSchema-instance}type", "CanonicalUser")
SubElement(grantee, "ID").text = grant.grantee
SubElement(grantee, "DisplayName").text = grant.grantee
SubElement(grant_el, "Permission").text = grant.permission
return _xml_response(root)
def _object_attributes_handler(bucket_name: str, object_key: str) -> Response:
if request.method != "GET":
return _method_not_allowed(["GET"])
principal, error = _object_principal("read", bucket_name, object_key)
if error:
return error
storage = _storage()
try:
path = storage.get_object_path(bucket_name, object_key)
file_stat = path.stat()
metadata = storage.get_object_metadata(bucket_name, object_key)
except (StorageError, FileNotFoundError):
return _error_response("NoSuchKey", "Object not found", 404)
requested = request.headers.get("x-amz-object-attributes", "")
attrs = {a.strip() for a in requested.split(",") if a.strip()}
root = Element("GetObjectAttributesResponse")
if "ETag" in attrs:
etag = metadata.get("__etag__") or storage._compute_etag(path)
SubElement(root, "ETag").text = etag
if "StorageClass" in attrs:
SubElement(root, "StorageClass").text = "STANDARD"
if "ObjectSize" in attrs:
SubElement(root, "ObjectSize").text = str(file_stat.st_size)
if "Checksum" in attrs:
SubElement(root, "Checksum")
if "ObjectParts" in attrs:
SubElement(root, "ObjectParts")
response = _xml_response(root)
response.headers["Last-Modified"] = http_date(file_stat.st_mtime)
return response
def _bucket_list_versions_handler(bucket_name: str) -> Response: def _bucket_list_versions_handler(bucket_name: str) -> Response:
"""Handle ListObjectVersions (GET /<bucket>?versions).""" """Handle ListObjectVersions (GET /<bucket>?versions)."""
if request.method != "GET": if request.method != "GET":
@@ -2360,6 +2481,10 @@ def _post_object(bucket_name: str) -> Response:
if success_action_redirect: if success_action_redirect:
allowed_hosts = current_app.config.get("ALLOWED_REDIRECT_HOSTS", []) allowed_hosts = current_app.config.get("ALLOWED_REDIRECT_HOSTS", [])
if not allowed_hosts: if not allowed_hosts:
current_app.logger.warning(
"ALLOWED_REDIRECT_HOSTS not configured, falling back to request Host header. "
"Set ALLOWED_REDIRECT_HOSTS for production deployments."
)
allowed_hosts = [request.host] allowed_hosts = [request.host]
parsed = urlparse(success_action_redirect) parsed = urlparse(success_action_redirect)
if parsed.scheme not in ("http", "https"): if parsed.scheme not in ("http", "https"):
@@ -2669,6 +2794,12 @@ def object_handler(bucket_name: str, object_key: str):
if "legal-hold" in request.args: if "legal-hold" in request.args:
return _object_legal_hold_handler(bucket_name, object_key) return _object_legal_hold_handler(bucket_name, object_key)
if "acl" in request.args:
return _object_acl_handler(bucket_name, object_key)
if "attributes" in request.args:
return _object_attributes_handler(bucket_name, object_key)
if request.method == "POST": if request.method == "POST":
if "uploads" in request.args: if "uploads" in request.args:
return _initiate_multipart_upload(bucket_name, object_key) return _initiate_multipart_upload(bucket_name, object_key)
@@ -2816,7 +2947,7 @@ def object_handler(bucket_name: str, object_key: str):
f.seek(start_pos) f.seek(start_pos)
remaining = length_to_read remaining = length_to_read
while remaining > 0: while remaining > 0:
chunk_size = min(65536, remaining) chunk_size = min(262144, remaining)
chunk = f.read(chunk_size) chunk = f.read(chunk_size)
if not chunk: if not chunk:
break break
@@ -2993,6 +3124,32 @@ def _bucket_policy_handler(bucket_name: str) -> Response:
return Response(status=204) return Response(status=204)
def _bucket_policy_status_handler(bucket_name: str) -> Response:
if request.method != "GET":
return _method_not_allowed(["GET"])
principal, error = _require_principal()
if error:
return error
try:
_authorize_action(principal, bucket_name, "policy")
except IamError as exc:
return _error_response("AccessDenied", str(exc), 403)
storage = _storage()
if not storage.bucket_exists(bucket_name):
return _error_response("NoSuchBucket", "Bucket does not exist", 404)
store = _bucket_policies()
policy = store.get_policy(bucket_name)
is_public = False
if policy:
for statement in policy.get("Statement", []):
if statement.get("Effect") == "Allow" and statement.get("Principal") == "*":
is_public = True
break
root = Element("PolicyStatus")
SubElement(root, "IsPublic").text = "TRUE" if is_public else "FALSE"
return _xml_response(root)
def _bucket_replication_handler(bucket_name: str) -> Response: def _bucket_replication_handler(bucket_name: str) -> Response:
if request.method not in {"GET", "PUT", "DELETE"}: if request.method not in {"GET", "PUT", "DELETE"}:
return _method_not_allowed(["GET", "PUT", "DELETE"]) return _method_not_allowed(["GET", "PUT", "DELETE"])
@@ -3206,7 +3363,7 @@ def head_object(bucket_name: str, object_key: str) -> Response:
path = _storage().get_object_path(bucket_name, object_key) path = _storage().get_object_path(bucket_name, object_key)
metadata = _storage().get_object_metadata(bucket_name, object_key) metadata = _storage().get_object_metadata(bucket_name, object_key)
stat = path.stat() stat = path.stat()
etag = _storage()._compute_etag(path) etag = metadata.get("__etag__") or _storage()._compute_etag(path)
response = Response(status=200) response = Response(status=200)
_apply_object_headers(response, file_stat=stat, metadata=metadata, etag=etag) _apply_object_headers(response, file_stat=stat, metadata=metadata, etag=etag)

View File

@@ -1,5 +1,6 @@
from __future__ import annotations from __future__ import annotations
import copy
import hashlib import hashlib
import json import json
import os import os
@@ -196,6 +197,8 @@ class ObjectStorage:
self._object_key_max_length_bytes = object_key_max_length_bytes self._object_key_max_length_bytes = object_key_max_length_bytes
self._sorted_key_cache: Dict[str, tuple[list[str], int]] = {} self._sorted_key_cache: Dict[str, tuple[list[str], int]] = {}
self._meta_index_locks: Dict[str, threading.Lock] = {} self._meta_index_locks: Dict[str, threading.Lock] = {}
self._meta_read_cache: OrderedDict[tuple, Optional[Dict[str, Any]]] = OrderedDict()
self._meta_read_cache_max = 2048
self._cleanup_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="ParentCleanup") self._cleanup_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="ParentCleanup")
def _get_bucket_lock(self, bucket_id: str) -> threading.Lock: def _get_bucket_lock(self, bucket_id: str) -> threading.Lock:
@@ -1904,14 +1907,38 @@ class ObjectStorage:
return self._meta_index_locks[index_path] return self._meta_index_locks[index_path]
def _read_index_entry(self, bucket_name: str, key: Path) -> Optional[Dict[str, Any]]: def _read_index_entry(self, bucket_name: str, key: Path) -> Optional[Dict[str, Any]]:
cache_key = (bucket_name, str(key))
with self._cache_lock:
hit = self._meta_read_cache.get(cache_key)
if hit is not None:
self._meta_read_cache.move_to_end(cache_key)
cached = hit[0]
return copy.deepcopy(cached) if cached is not None else None
index_path, entry_name = self._index_file_for_key(bucket_name, key) index_path, entry_name = self._index_file_for_key(bucket_name, key)
if _HAS_RUST:
result = _rc.read_index_entry(str(index_path), entry_name)
else:
if not index_path.exists(): if not index_path.exists():
return None result = None
else:
try: try:
index_data = json.loads(index_path.read_text(encoding="utf-8")) index_data = json.loads(index_path.read_text(encoding="utf-8"))
return index_data.get(entry_name) result = index_data.get(entry_name)
except (OSError, json.JSONDecodeError): except (OSError, json.JSONDecodeError):
return None result = None
with self._cache_lock:
while len(self._meta_read_cache) >= self._meta_read_cache_max:
self._meta_read_cache.popitem(last=False)
self._meta_read_cache[cache_key] = (copy.deepcopy(result) if result is not None else None,)
return result
def _invalidate_meta_read_cache(self, bucket_name: str, key: Path) -> None:
cache_key = (bucket_name, str(key))
with self._cache_lock:
self._meta_read_cache.pop(cache_key, None)
def _write_index_entry(self, bucket_name: str, key: Path, entry: Dict[str, Any]) -> None: def _write_index_entry(self, bucket_name: str, key: Path, entry: Dict[str, Any]) -> None:
index_path, entry_name = self._index_file_for_key(bucket_name, key) index_path, entry_name = self._index_file_for_key(bucket_name, key)
@@ -1926,16 +1953,19 @@ class ObjectStorage:
pass pass
index_data[entry_name] = entry index_data[entry_name] = entry
index_path.write_text(json.dumps(index_data), encoding="utf-8") index_path.write_text(json.dumps(index_data), encoding="utf-8")
self._invalidate_meta_read_cache(bucket_name, key)
def _delete_index_entry(self, bucket_name: str, key: Path) -> None: def _delete_index_entry(self, bucket_name: str, key: Path) -> None:
index_path, entry_name = self._index_file_for_key(bucket_name, key) index_path, entry_name = self._index_file_for_key(bucket_name, key)
if not index_path.exists(): if not index_path.exists():
self._invalidate_meta_read_cache(bucket_name, key)
return return
lock = self._get_meta_index_lock(str(index_path)) lock = self._get_meta_index_lock(str(index_path))
with lock: with lock:
try: try:
index_data = json.loads(index_path.read_text(encoding="utf-8")) index_data = json.loads(index_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError): except (OSError, json.JSONDecodeError):
self._invalidate_meta_read_cache(bucket_name, key)
return return
if entry_name in index_data: if entry_name in index_data:
del index_data[entry_name] del index_data[entry_name]
@@ -1946,6 +1976,7 @@ class ObjectStorage:
index_path.unlink() index_path.unlink()
except OSError: except OSError:
pass pass
self._invalidate_meta_read_cache(bucket_name, key)
def _normalize_metadata(self, metadata: Optional[Dict[str, str]]) -> Optional[Dict[str, str]]: def _normalize_metadata(self, metadata: Optional[Dict[str, str]]) -> Optional[Dict[str, str]]:
if not metadata: if not metadata:

View File

@@ -508,11 +508,15 @@ def bucket_detail(bucket_name: str):
can_manage_quota = is_replication_admin can_manage_quota = is_replication_admin
website_config = None website_config = None
website_domains = []
if website_hosting_enabled: if website_hosting_enabled:
try: try:
website_config = storage.get_bucket_website(bucket_name) website_config = storage.get_bucket_website(bucket_name)
except StorageError: except StorageError:
website_config = None website_config = None
domain_store = current_app.extensions.get("website_domains")
if domain_store:
website_domains = domain_store.get_domains_for_bucket(bucket_name)
objects_api_url = url_for("ui.list_bucket_objects", bucket_name=bucket_name) objects_api_url = url_for("ui.list_bucket_objects", bucket_name=bucket_name)
objects_stream_url = url_for("ui.stream_bucket_objects", bucket_name=bucket_name) objects_stream_url = url_for("ui.stream_bucket_objects", bucket_name=bucket_name)
@@ -558,6 +562,7 @@ def bucket_detail(bucket_name: str):
site_sync_enabled=site_sync_enabled, site_sync_enabled=site_sync_enabled,
website_hosting_enabled=website_hosting_enabled, website_hosting_enabled=website_hosting_enabled,
website_config=website_config, website_config=website_config,
website_domains=website_domains,
can_manage_website=can_edit_policy, can_manage_website=can_edit_policy,
) )
@@ -738,7 +743,6 @@ def initiate_multipart_upload(bucket_name: str):
@ui_bp.put("/buckets/<bucket_name>/multipart/<upload_id>/parts") @ui_bp.put("/buckets/<bucket_name>/multipart/<upload_id>/parts")
@limiter.exempt
@csrf.exempt @csrf.exempt
def upload_multipart_part(bucket_name: str, upload_id: str): def upload_multipart_part(bucket_name: str, upload_id: str):
principal = _current_principal() principal = _current_principal()
@@ -2374,7 +2378,10 @@ def website_domains_dashboard():
store = current_app.extensions.get("website_domains") store = current_app.extensions.get("website_domains")
mappings = store.list_all() if store else [] mappings = store.list_all() if store else []
storage = _storage() storage = _storage()
buckets = [b.name for b in storage.list_buckets()] buckets = [
b.name for b in storage.list_buckets()
if storage.get_bucket_website(b.name)
]
return render_template( return render_template(
"website_domains.html", "website_domains.html",
mappings=mappings, mappings=mappings,
@@ -3293,9 +3300,12 @@ def sites_dashboard():
@ui_bp.post("/sites/local") @ui_bp.post("/sites/local")
def update_local_site(): def update_local_site():
principal = _current_principal() principal = _current_principal()
wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest"
try: try:
_iam().authorize(principal, None, "iam:*") _iam().authorize(principal, None, "iam:*")
except IamError: except IamError:
if wants_json:
return jsonify({"error": "Access denied"}), 403
flash("Access denied", "danger") flash("Access denied", "danger")
return redirect(url_for("ui.sites_dashboard")) return redirect(url_for("ui.sites_dashboard"))
@@ -3306,6 +3316,8 @@ def update_local_site():
display_name = request.form.get("display_name", "").strip() display_name = request.form.get("display_name", "").strip()
if not site_id: if not site_id:
if wants_json:
return jsonify({"error": "Site ID is required"}), 400
flash("Site ID is required", "danger") flash("Site ID is required", "danger")
return redirect(url_for("ui.sites_dashboard")) return redirect(url_for("ui.sites_dashboard"))
@@ -3327,6 +3339,8 @@ def update_local_site():
) )
registry.set_local_site(site) registry.set_local_site(site)
if wants_json:
return jsonify({"message": "Local site configuration updated"})
flash("Local site configuration updated", "success") flash("Local site configuration updated", "success")
return redirect(url_for("ui.sites_dashboard")) return redirect(url_for("ui.sites_dashboard"))
@@ -3334,9 +3348,12 @@ def update_local_site():
@ui_bp.post("/sites/peers") @ui_bp.post("/sites/peers")
def add_peer_site(): def add_peer_site():
principal = _current_principal() principal = _current_principal()
wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest"
try: try:
_iam().authorize(principal, None, "iam:*") _iam().authorize(principal, None, "iam:*")
except IamError: except IamError:
if wants_json:
return jsonify({"error": "Access denied"}), 403
flash("Access denied", "danger") flash("Access denied", "danger")
return redirect(url_for("ui.sites_dashboard")) return redirect(url_for("ui.sites_dashboard"))
@@ -3348,9 +3365,13 @@ def add_peer_site():
connection_id = request.form.get("connection_id", "").strip() or None connection_id = request.form.get("connection_id", "").strip() or None
if not site_id: if not site_id:
if wants_json:
return jsonify({"error": "Site ID is required"}), 400
flash("Site ID is required", "danger") flash("Site ID is required", "danger")
return redirect(url_for("ui.sites_dashboard")) return redirect(url_for("ui.sites_dashboard"))
if not endpoint: if not endpoint:
if wants_json:
return jsonify({"error": "Endpoint is required"}), 400
flash("Endpoint is required", "danger") flash("Endpoint is required", "danger")
return redirect(url_for("ui.sites_dashboard")) return redirect(url_for("ui.sites_dashboard"))
@@ -3362,10 +3383,14 @@ def add_peer_site():
registry = _site_registry() registry = _site_registry()
if registry.get_peer(site_id): if registry.get_peer(site_id):
if wants_json:
return jsonify({"error": f"Peer site '{site_id}' already exists"}), 409
flash(f"Peer site '{site_id}' already exists", "danger") flash(f"Peer site '{site_id}' already exists", "danger")
return redirect(url_for("ui.sites_dashboard")) return redirect(url_for("ui.sites_dashboard"))
if connection_id and not _connections().get(connection_id): if connection_id and not _connections().get(connection_id):
if wants_json:
return jsonify({"error": f"Connection '{connection_id}' not found"}), 404
flash(f"Connection '{connection_id}' not found", "danger") flash(f"Connection '{connection_id}' not found", "danger")
return redirect(url_for("ui.sites_dashboard")) return redirect(url_for("ui.sites_dashboard"))
@@ -3379,6 +3404,11 @@ def add_peer_site():
) )
registry.add_peer(peer) registry.add_peer(peer)
if wants_json:
redirect_url = None
if connection_id:
redirect_url = url_for("ui.replication_wizard", site_id=site_id)
return jsonify({"message": f"Peer site '{site_id}' added", "redirect": redirect_url})
flash(f"Peer site '{site_id}' added", "success") flash(f"Peer site '{site_id}' added", "success")
if connection_id: if connection_id:
@@ -3389,9 +3419,12 @@ def add_peer_site():
@ui_bp.post("/sites/peers/<site_id>/update") @ui_bp.post("/sites/peers/<site_id>/update")
def update_peer_site(site_id: str): def update_peer_site(site_id: str):
principal = _current_principal() principal = _current_principal()
wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest"
try: try:
_iam().authorize(principal, None, "iam:*") _iam().authorize(principal, None, "iam:*")
except IamError: except IamError:
if wants_json:
return jsonify({"error": "Access denied"}), 403
flash("Access denied", "danger") flash("Access denied", "danger")
return redirect(url_for("ui.sites_dashboard")) return redirect(url_for("ui.sites_dashboard"))
@@ -3399,6 +3432,8 @@ def update_peer_site(site_id: str):
existing = registry.get_peer(site_id) existing = registry.get_peer(site_id)
if not existing: if not existing:
if wants_json:
return jsonify({"error": f"Peer site '{site_id}' not found"}), 404
flash(f"Peer site '{site_id}' not found", "danger") flash(f"Peer site '{site_id}' not found", "danger")
return redirect(url_for("ui.sites_dashboard")) return redirect(url_for("ui.sites_dashboard"))
@@ -3406,7 +3441,10 @@ def update_peer_site(site_id: str):
region = request.form.get("region", existing.region).strip() region = request.form.get("region", existing.region).strip()
priority = request.form.get("priority", str(existing.priority)) priority = request.form.get("priority", str(existing.priority))
display_name = request.form.get("display_name", existing.display_name).strip() display_name = request.form.get("display_name", existing.display_name).strip()
connection_id = request.form.get("connection_id", "").strip() or existing.connection_id if "connection_id" in request.form:
connection_id = request.form["connection_id"].strip() or None
else:
connection_id = existing.connection_id
try: try:
priority_int = int(priority) priority_int = int(priority)
@@ -3414,6 +3452,8 @@ def update_peer_site(site_id: str):
priority_int = existing.priority priority_int = existing.priority
if connection_id and not _connections().get(connection_id): if connection_id and not _connections().get(connection_id):
if wants_json:
return jsonify({"error": f"Connection '{connection_id}' not found"}), 404
flash(f"Connection '{connection_id}' not found", "danger") flash(f"Connection '{connection_id}' not found", "danger")
return redirect(url_for("ui.sites_dashboard")) return redirect(url_for("ui.sites_dashboard"))
@@ -3430,6 +3470,8 @@ def update_peer_site(site_id: str):
) )
registry.update_peer(peer) registry.update_peer(peer)
if wants_json:
return jsonify({"message": f"Peer site '{site_id}' updated"})
flash(f"Peer site '{site_id}' updated", "success") flash(f"Peer site '{site_id}' updated", "success")
return redirect(url_for("ui.sites_dashboard")) return redirect(url_for("ui.sites_dashboard"))
@@ -3437,16 +3479,23 @@ def update_peer_site(site_id: str):
@ui_bp.post("/sites/peers/<site_id>/delete") @ui_bp.post("/sites/peers/<site_id>/delete")
def delete_peer_site(site_id: str): def delete_peer_site(site_id: str):
principal = _current_principal() principal = _current_principal()
wants_json = request.headers.get("X-Requested-With") == "XMLHttpRequest"
try: try:
_iam().authorize(principal, None, "iam:*") _iam().authorize(principal, None, "iam:*")
except IamError: except IamError:
if wants_json:
return jsonify({"error": "Access denied"}), 403
flash("Access denied", "danger") flash("Access denied", "danger")
return redirect(url_for("ui.sites_dashboard")) return redirect(url_for("ui.sites_dashboard"))
registry = _site_registry() registry = _site_registry()
if registry.delete_peer(site_id): if registry.delete_peer(site_id):
if wants_json:
return jsonify({"message": f"Peer site '{site_id}' deleted"})
flash(f"Peer site '{site_id}' deleted", "success") flash(f"Peer site '{site_id}' deleted", "success")
else: else:
if wants_json:
return jsonify({"error": f"Peer site '{site_id}' not found"}), 404
flash(f"Peer site '{site_id}' not found", "danger") flash(f"Peer site '{site_id}' not found", "danger")
return redirect(url_for("ui.sites_dashboard")) return redirect(url_for("ui.sites_dashboard"))

View File

@@ -35,13 +35,16 @@ class WebsiteDomainStore:
self.config_path = config_path self.config_path = config_path
self._lock = threading.Lock() self._lock = threading.Lock()
self._domains: Dict[str, str] = {} self._domains: Dict[str, str] = {}
self._last_mtime: float = 0.0
self.reload() self.reload()
def reload(self) -> None: def reload(self) -> None:
if not self.config_path.exists(): if not self.config_path.exists():
self._domains = {} self._domains = {}
self._last_mtime = 0.0
return return
try: try:
self._last_mtime = self.config_path.stat().st_mtime
with open(self.config_path, "r", encoding="utf-8") as f: with open(self.config_path, "r", encoding="utf-8") as f:
data = json.load(f) data = json.load(f)
if isinstance(data, dict): if isinstance(data, dict):
@@ -51,19 +54,45 @@ class WebsiteDomainStore:
except (OSError, json.JSONDecodeError): except (OSError, json.JSONDecodeError):
self._domains = {} self._domains = {}
def _maybe_reload(self) -> None:
try:
if self.config_path.exists():
mtime = self.config_path.stat().st_mtime
if mtime != self._last_mtime:
self._last_mtime = mtime
with open(self.config_path, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, dict):
self._domains = {k.lower(): v for k, v in data.items()}
else:
self._domains = {}
elif self._domains:
self._domains = {}
self._last_mtime = 0.0
except (OSError, json.JSONDecodeError):
pass
def _save(self) -> None: def _save(self) -> None:
self.config_path.parent.mkdir(parents=True, exist_ok=True) self.config_path.parent.mkdir(parents=True, exist_ok=True)
with open(self.config_path, "w", encoding="utf-8") as f: with open(self.config_path, "w", encoding="utf-8") as f:
json.dump(self._domains, f, indent=2) json.dump(self._domains, f, indent=2)
self._last_mtime = self.config_path.stat().st_mtime
def list_all(self) -> List[Dict[str, str]]: def list_all(self) -> List[Dict[str, str]]:
with self._lock: with self._lock:
self._maybe_reload()
return [{"domain": d, "bucket": b} for d, b in self._domains.items()] return [{"domain": d, "bucket": b} for d, b in self._domains.items()]
def get_bucket(self, domain: str) -> Optional[str]: def get_bucket(self, domain: str) -> Optional[str]:
with self._lock: with self._lock:
self._maybe_reload()
return self._domains.get(domain.lower()) return self._domains.get(domain.lower())
def get_domains_for_bucket(self, bucket: str) -> List[str]:
with self._lock:
self._maybe_reload()
return [d for d, b in self._domains.items() if b == bucket]
def set_mapping(self, domain: str, bucket: str) -> None: def set_mapping(self, domain: str, bucket: str) -> None:
with self._lock: with self._lock:
self._domains[domain.lower()] = bucket self._domains[domain.lower()] = bucket

421
myfsio_core/Cargo.lock generated
View File

@@ -1,421 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "aho-corasick"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
dependencies = [
"memchr",
]
[[package]]
name = "allocator-api2"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
[[package]]
name = "bitflags"
version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
[[package]]
name = "block-buffer"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array",
]
[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "cpufeatures"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
dependencies = [
"libc",
]
[[package]]
name = "crypto-common"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
"subtle",
]
[[package]]
name = "equivalent"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "generic-array"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "hashbrown"
version = "0.15.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash",
]
[[package]]
name = "heck"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "hmac"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
dependencies = [
"digest",
]
[[package]]
name = "libc"
version = "0.2.182"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
[[package]]
name = "lock_api"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
dependencies = [
"scopeguard",
]
[[package]]
name = "lru"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f8cc7106155f10bdf99a6f379688f543ad6596a415375b36a59a054ceda1198"
dependencies = [
"hashbrown",
]
[[package]]
name = "md-5"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
dependencies = [
"cfg-if",
"digest",
]
[[package]]
name = "memchr"
version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
[[package]]
name = "myfsio_core"
version = "0.1.0"
dependencies = [
"hex",
"hmac",
"lru",
"md-5",
"parking_lot",
"pyo3",
"regex",
"sha2",
"unicode-normalization",
]
[[package]]
name = "once_cell"
version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "parking_lot"
version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-link",
]
[[package]]
name = "portable-atomic"
version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
[[package]]
name = "proc-macro2"
version = "1.0.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
dependencies = [
"unicode-ident",
]
[[package]]
name = "pyo3"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c738662e2181be11cb82487628404254902bb3225d8e9e99c31f3ef82a405c"
dependencies = [
"libc",
"once_cell",
"portable-atomic",
"pyo3-build-config",
"pyo3-ffi",
"pyo3-macros",
]
[[package]]
name = "pyo3-build-config"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9ca0864a7dd3c133a7f3f020cbff2e12e88420da854c35540fd20ce2d60e435"
dependencies = [
"target-lexicon",
]
[[package]]
name = "pyo3-ffi"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dfc1956b709823164763a34cc42bbfd26b8730afa77809a3df8b94a3ae3b059"
dependencies = [
"libc",
"pyo3-build-config",
]
[[package]]
name = "pyo3-macros"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29dc660ad948bae134d579661d08033fbb1918f4529c3bbe3257a68f2009ddf2"
dependencies = [
"proc-macro2",
"pyo3-macros-backend",
"quote",
"syn",
]
[[package]]
name = "pyo3-macros-backend"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78cd6c6d718acfcedf26c3d21fe0f053624368b0d44298c55d7138fde9331f7"
dependencies = [
"heck",
"proc-macro2",
"pyo3-build-config",
"quote",
"syn",
]
[[package]]
name = "quote"
version = "1.0.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
dependencies = [
"proc-macro2",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
dependencies = [
"bitflags",
]
[[package]]
name = "regex"
version = "1.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c"
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "sha2"
version = "0.10.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "smallvec"
version = "1.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
[[package]]
name = "subtle"
version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "syn"
version = "2.0.116"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3df424c70518695237746f84cede799c9c58fcb37450d7b23716568cc8bc69cb"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "target-lexicon"
version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
[[package]]
name = "tinyvec"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "typenum"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "unicode-normalization"
version = "0.1.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
dependencies = [
"tinyvec",
]
[[package]]
name = "version_check"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"

View File

@@ -14,6 +14,8 @@ sha2 = "0.10"
md-5 = "0.10" md-5 = "0.10"
hex = "0.4" hex = "0.4"
unicode-normalization = "0.1" unicode-normalization = "0.1"
serde_json = "1"
regex = "1" regex = "1"
lru = "0.14" lru = "0.14"
parking_lot = "0.12" parking_lot = "0.12"
percent-encoding = "2"

View File

@@ -1,4 +1,5 @@
mod hashing; mod hashing;
mod metadata;
mod sigv4; mod sigv4;
mod validation; mod validation;
@@ -10,6 +11,7 @@ mod myfsio_core {
#[pymodule_init] #[pymodule_init]
fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { fn init(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_function(wrap_pyfunction!(sigv4::verify_sigv4_signature, m)?)?;
m.add_function(wrap_pyfunction!(sigv4::derive_signing_key, m)?)?; m.add_function(wrap_pyfunction!(sigv4::derive_signing_key, m)?)?;
m.add_function(wrap_pyfunction!(sigv4::compute_signature, m)?)?; m.add_function(wrap_pyfunction!(sigv4::compute_signature, m)?)?;
m.add_function(wrap_pyfunction!(sigv4::build_string_to_sign, m)?)?; m.add_function(wrap_pyfunction!(sigv4::build_string_to_sign, m)?)?;
@@ -25,6 +27,8 @@ mod myfsio_core {
m.add_function(wrap_pyfunction!(validation::validate_object_key, m)?)?; m.add_function(wrap_pyfunction!(validation::validate_object_key, m)?)?;
m.add_function(wrap_pyfunction!(validation::validate_bucket_name, m)?)?; m.add_function(wrap_pyfunction!(validation::validate_bucket_name, m)?)?;
m.add_function(wrap_pyfunction!(metadata::read_index_entry, m)?)?;
Ok(()) Ok(())
} }
} }

View File

@@ -0,0 +1,71 @@
use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyList, PyString};
use serde_json::Value;
use std::fs;
const MAX_DEPTH: u32 = 64;
fn value_to_py(py: Python<'_>, v: &Value, depth: u32) -> PyResult<Py<PyAny>> {
if depth > MAX_DEPTH {
return Err(PyValueError::new_err("JSON nesting too deep"));
}
match v {
Value::Null => Ok(py.None()),
Value::Bool(b) => Ok((*b).into_pyobject(py)?.to_owned().into_any().unbind()),
Value::Number(n) => {
if let Some(i) = n.as_i64() {
Ok(i.into_pyobject(py)?.into_any().unbind())
} else if let Some(f) = n.as_f64() {
Ok(f.into_pyobject(py)?.into_any().unbind())
} else {
Ok(py.None())
}
}
Value::String(s) => Ok(PyString::new(py, s).into_any().unbind()),
Value::Array(arr) => {
let list = PyList::empty(py);
for item in arr {
list.append(value_to_py(py, item, depth + 1)?)?;
}
Ok(list.into_any().unbind())
}
Value::Object(map) => {
let dict = PyDict::new(py);
for (k, val) in map {
dict.set_item(k, value_to_py(py, val, depth + 1)?)?;
}
Ok(dict.into_any().unbind())
}
}
}
#[pyfunction]
pub fn read_index_entry(
py: Python<'_>,
path: &str,
entry_name: &str,
) -> PyResult<Option<Py<PyAny>>> {
let path_owned = path.to_owned();
let entry_owned = entry_name.to_owned();
let entry: Option<Value> = py.detach(move || -> PyResult<Option<Value>> {
let content = match fs::read_to_string(&path_owned) {
Ok(c) => c,
Err(_) => return Ok(None),
};
let parsed: Value = match serde_json::from_str(&content) {
Ok(v) => v,
Err(_) => return Ok(None),
};
match parsed {
Value::Object(mut map) => Ok(map.remove(&entry_owned)),
_ => Ok(None),
}
})?;
match entry {
Some(val) => Ok(Some(value_to_py(py, &val, 0)?)),
None => Ok(None),
}
}

View File

@@ -1,6 +1,7 @@
use hmac::{Hmac, Mac}; use hmac::{Hmac, Mac};
use lru::LruCache; use lru::LruCache;
use parking_lot::Mutex; use parking_lot::Mutex;
use percent_encoding::{percent_encode, AsciiSet, NON_ALPHANUMERIC};
use pyo3::prelude::*; use pyo3::prelude::*;
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use std::num::NonZeroUsize; use std::num::NonZeroUsize;
@@ -19,14 +20,29 @@ static SIGNING_KEY_CACHE: LazyLock<Mutex<LruCache<(String, String, String, Strin
const CACHE_TTL_SECS: u64 = 60; const CACHE_TTL_SECS: u64 = 60;
const AWS_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC
.remove(b'-')
.remove(b'_')
.remove(b'.')
.remove(b'~');
fn hmac_sha256(key: &[u8], msg: &[u8]) -> Vec<u8> { fn hmac_sha256(key: &[u8], msg: &[u8]) -> Vec<u8> {
let mut mac = HmacSha256::new_from_slice(key).expect("HMAC key length is always valid"); let mut mac = HmacSha256::new_from_slice(key).expect("HMAC key length is always valid");
mac.update(msg); mac.update(msg);
mac.finalize().into_bytes().to_vec() mac.finalize().into_bytes().to_vec()
} }
#[pyfunction] fn sha256_hex(data: &[u8]) -> String {
pub fn derive_signing_key( let mut hasher = Sha256::new();
hasher.update(data);
hex::encode(hasher.finalize())
}
fn aws_uri_encode(input: &str) -> String {
percent_encode(input.as_bytes(), AWS_ENCODE_SET).to_string()
}
fn derive_signing_key_cached(
secret_key: &str, secret_key: &str,
date_stamp: &str, date_stamp: &str,
region: &str, region: &str,
@@ -68,18 +84,91 @@ pub fn derive_signing_key(
k_signing k_signing
} }
fn constant_time_compare_inner(a: &[u8], b: &[u8]) -> bool {
if a.len() != b.len() {
return false;
}
let mut result: u8 = 0;
for (x, y) in a.iter().zip(b.iter()) {
result |= x ^ y;
}
result == 0
}
#[pyfunction]
pub fn verify_sigv4_signature(
method: &str,
canonical_uri: &str,
query_params: Vec<(String, String)>,
signed_headers_str: &str,
header_values: Vec<(String, String)>,
payload_hash: &str,
amz_date: &str,
date_stamp: &str,
region: &str,
service: &str,
secret_key: &str,
provided_signature: &str,
) -> bool {
let mut sorted_params = query_params;
sorted_params.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)));
let canonical_query_string = sorted_params
.iter()
.map(|(k, v)| format!("{}={}", aws_uri_encode(k), aws_uri_encode(v)))
.collect::<Vec<_>>()
.join("&");
let mut canonical_headers = String::new();
for (name, value) in &header_values {
let lower_name = name.to_lowercase();
let normalized = value.split_whitespace().collect::<Vec<_>>().join(" ");
let final_value = if lower_name == "expect" && normalized.is_empty() {
"100-continue"
} else {
&normalized
};
canonical_headers.push_str(&lower_name);
canonical_headers.push(':');
canonical_headers.push_str(final_value);
canonical_headers.push('\n');
}
let canonical_request = format!(
"{}\n{}\n{}\n{}\n{}\n{}",
method, canonical_uri, canonical_query_string, canonical_headers, signed_headers_str, payload_hash
);
let credential_scope = format!("{}/{}/{}/aws4_request", date_stamp, region, service);
let cr_hash = sha256_hex(canonical_request.as_bytes());
let string_to_sign = format!(
"AWS4-HMAC-SHA256\n{}\n{}\n{}",
amz_date, credential_scope, cr_hash
);
let signing_key = derive_signing_key_cached(secret_key, date_stamp, region, service);
let calculated = hmac_sha256(&signing_key, string_to_sign.as_bytes());
let calculated_hex = hex::encode(&calculated);
constant_time_compare_inner(calculated_hex.as_bytes(), provided_signature.as_bytes())
}
#[pyfunction]
pub fn derive_signing_key(
secret_key: &str,
date_stamp: &str,
region: &str,
service: &str,
) -> Vec<u8> {
derive_signing_key_cached(secret_key, date_stamp, region, service)
}
#[pyfunction] #[pyfunction]
pub fn compute_signature(signing_key: &[u8], string_to_sign: &str) -> String { pub fn compute_signature(signing_key: &[u8], string_to_sign: &str) -> String {
let sig = hmac_sha256(signing_key, string_to_sign.as_bytes()); let sig = hmac_sha256(signing_key, string_to_sign.as_bytes());
hex::encode(sig) hex::encode(sig)
} }
fn sha256_hex(data: &[u8]) -> String {
let mut hasher = Sha256::new();
hasher.update(data);
hex::encode(hasher.finalize())
}
#[pyfunction] #[pyfunction]
pub fn build_string_to_sign( pub fn build_string_to_sign(
amz_date: &str, amz_date: &str,
@@ -87,19 +176,15 @@ pub fn build_string_to_sign(
canonical_request: &str, canonical_request: &str,
) -> String { ) -> String {
let cr_hash = sha256_hex(canonical_request.as_bytes()); let cr_hash = sha256_hex(canonical_request.as_bytes());
format!("AWS4-HMAC-SHA256\n{}\n{}\n{}", amz_date, credential_scope, cr_hash) format!(
"AWS4-HMAC-SHA256\n{}\n{}\n{}",
amz_date, credential_scope, cr_hash
)
} }
#[pyfunction] #[pyfunction]
pub fn constant_time_compare(a: &str, b: &str) -> bool { pub fn constant_time_compare(a: &str, b: &str) -> bool {
if a.len() != b.len() { constant_time_compare_inner(a.as_bytes(), b.as_bytes())
return false;
}
let mut result: u8 = 0;
for (x, y) in a.bytes().zip(b.bytes()) {
result |= x ^ y;
}
result == 0
} }
#[pyfunction] #[pyfunction]

View File

@@ -1 +0,0 @@
{"rustc_fingerprint":13172970000770725120,"outputs":{"7971740275564407648":{"success":true,"status":"","code":0,"stdout":"___.exe\nlib___.rlib\n___.dll\n___.dll\n___.lib\n___.dll\nC:\\Users\\jun\\.rustup\\toolchains\\stable-x86_64-pc-windows-msvc\npacked\n___\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_abi=\"\"\ntarget_arch=\"x86_64\"\ntarget_endian=\"little\"\ntarget_env=\"msvc\"\ntarget_family=\"windows\"\ntarget_feature=\"cmpxchg16b\"\ntarget_feature=\"fxsr\"\ntarget_feature=\"sse\"\ntarget_feature=\"sse2\"\ntarget_feature=\"sse3\"\ntarget_has_atomic=\"128\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_os=\"windows\"\ntarget_pointer_width=\"64\"\ntarget_vendor=\"pc\"\nwindows\n","stderr":""},"17747080675513052775":{"success":true,"status":"","code":0,"stdout":"rustc 1.93.1 (01f6ddf75 2026-02-11)\nbinary: rustc\ncommit-hash: 01f6ddf7588f42ae2d7eb0a2f21d44e8e96674cf\ncommit-date: 2026-02-11\nhost: x86_64-pc-windows-msvc\nrelease: 1.93.1\nLLVM version: 21.1.8\n","stderr":""}},"successes":{}}

View File

@@ -1,3 +0,0 @@
Signature: 8a477f597d28d172789f06886806bc55
# This file is a cache directory tag created by cargo.
# For information about cache directory tags see https://bford.info/cachedir/

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[\"perf-literal\", \"std\"]","declared_features":"[\"default\", \"logging\", \"perf-literal\", \"std\"]","target":7534583537114156500,"profile":2040997289075261528,"path":6364296192483896971,"deps":[[1363051979936526615,"memchr",false,11090220145123168660]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\aho-corasick-45694771b543be75\\dep-lib-aho_corasick","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[\"alloc\"]","declared_features":"[\"alloc\", \"default\", \"fresh-rust\", \"nightly\", \"serde\", \"std\"]","target":5388200169723499962,"profile":4067574213046180398,"path":10654049299693593327,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\allocator-api2-db7934dbe96de5b4\\dep-lib-allocator_api2","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":6962977057026645649,"profile":1369601567987815722,"path":9853093265219907461,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\autocfg-1c4fb7a37cc3df69\\dep-lib-autocfg","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":4098124618827574291,"profile":2040997289075261528,"path":3658007358608479489,"deps":[[10520923840501062997,"generic_array",false,11555283918993371487]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\block-buffer-95b0ac364bec72f9\\dep-lib-block_buffer","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[\"core\", \"rustc-dep-of-std\"]","target":13840298032947503755,"profile":2040997289075261528,"path":4093486168504982869,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\cfg-if-be2711f84a777e73\\dep-lib-cfg_if","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":2330704043955282025,"profile":2040997289075261528,"path":13200428550696548327,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\cpufeatures-980094f8735c42d1\\dep-lib-cpufeatures","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[\"std\"]","declared_features":"[\"getrandom\", \"rand_core\", \"std\"]","target":12082577455412410174,"profile":2040997289075261528,"path":14902376638882023040,"deps":[[857979250431893282,"typenum",false,7416411392359930020],[10520923840501062997,"generic_array",false,11555283918993371487]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\crypto-common-289a508abdda3048\\dep-lib-crypto_common","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[\"alloc\", \"block-buffer\", \"core-api\", \"default\", \"mac\", \"std\", \"subtle\"]","declared_features":"[\"alloc\", \"blobby\", \"block-buffer\", \"const-oid\", \"core-api\", \"default\", \"dev\", \"mac\", \"oid\", \"rand_core\", \"std\", \"subtle\"]","target":7510122432137863311,"profile":2040997289075261528,"path":11503432597517024930,"deps":[[6039282458970808711,"crypto_common",false,11252724541433210505],[10626340395483396037,"block_buffer",false,17139625223017709343],[17003143334332120809,"subtle",false,8597342066671925934]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\digest-a91458bfa5613332\\dep-lib-digest","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":1524667692659508025,"profile":2040997289075261528,"path":17534356223679657546,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\equivalent-943ac856871c0988\\dep-lib-equivalent","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[\"default\", \"std\"]","target":18077926938045032029,"profile":2040997289075261528,"path":9869209539952544870,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\foldhash-b8a92f8c10d550f7\\dep-lib-foldhash","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[\"more_lengths\"]","declared_features":"[\"more_lengths\", \"serde\", \"zeroize\"]","target":12318548087768197662,"profile":1369601567987815722,"path":13853454403963664247,"deps":[[5398981501050481332,"version_check",false,16419025953046340415]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\generic-array-2462daa120fe5936\\dep-build-script-build-script-build","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[\"more_lengths\"]","declared_features":"[\"more_lengths\", \"serde\", \"zeroize\"]","target":13084005262763373425,"profile":2040997289075261528,"path":12463275850883329568,"deps":[[857979250431893282,"typenum",false,7416411392359930020],[10520923840501062997,"build_script_build",false,16977603856295925732]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\generic-array-62216349963f3a3c\\dep-lib-generic_array","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"","declared_features":"","target":0,"profile":0,"path":0,"deps":[[10520923840501062997,"build_script_build",false,464306762232604144]],"local":[{"Precalculated":"0.14.7"}],"rustflags":[],"config":0,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[\"allocator-api2\", \"default\", \"default-hasher\", \"equivalent\", \"inline-more\", \"raw-entry\"]","declared_features":"[\"alloc\", \"allocator-api2\", \"core\", \"default\", \"default-hasher\", \"equivalent\", \"inline-more\", \"nightly\", \"raw-entry\", \"rayon\", \"rustc-dep-of-std\", \"rustc-internal-api\", \"serde\"]","target":13796197676120832388,"profile":2040997289075261528,"path":12448322139402656924,"deps":[[5230392855116717286,"equivalent",false,6042941999404782907],[9150530836556604396,"allocator_api2",false,16398368410642502979],[10842263908529601448,"foldhash",false,10953695263156452023]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\hashbrown-510d641b592c306b\\dep-lib-hashbrown","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":17886154901722686619,"profile":1369601567987815722,"path":8608102977929876445,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\heck-b47c94fd2a7e00cb\\dep-lib-heck","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
41890ebff4143fa5

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[\"alloc\", \"default\", \"std\"]","declared_features":"[\"alloc\", \"default\", \"serde\", \"std\"]","target":4242469766639956503,"profile":2040997289075261528,"path":6793865871540733919,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\hex-253414d2260adcdf\\dep-lib-hex","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[\"reset\", \"std\"]","target":12991177224612424488,"profile":2040997289075261528,"path":17893893568771568113,"deps":[[17475753849556516473,"digest",false,15621022965039188625]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\hmac-3297e61b9effb758\\dep-lib-hmac","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[]","declared_features":"[]","target":8726396592336845528,"profile":1369601567987815722,"path":18304219166357541938,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\indoc-0c686c3f403a2566\\dep-lib-indoc","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[\"default\", \"std\"]","declared_features":"[\"align\", \"const-extern-fn\", \"default\", \"extra_traits\", \"rustc-dep-of-std\", \"rustc-std-workspace-core\", \"std\", \"use_std\"]","target":17682796336736096309,"profile":7322064999780386650,"path":3108645287704295931,"deps":[[18365559012052052344,"build_script_build",false,12197584826291254217]],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\libc-5f9f280eeaad3bb3\\dep-lib-libc","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"[\"default\", \"std\"]","declared_features":"[\"align\", \"const-extern-fn\", \"default\", \"extra_traits\", \"rustc-dep-of-std\", \"rustc-std-workspace-core\", \"std\", \"use_std\"]","target":5408242616063297496,"profile":8928907579149787682,"path":7198683120865577851,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release\\.fingerprint\\libc-70cd639287284bb6\\dep-build-script-build-script-build","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

View File

@@ -1 +0,0 @@
{"rustc":8323788817864214825,"features":"","declared_features":"","target":0,"profile":0,"path":0,"deps":[[18365559012052052344,"build_script_build",false,2720603730513905790]],"local":[{"RerunIfChanged":{"output":"release\\build\\libc-bf9c887cc2b82f5a\\output","paths":["build.rs"]}},{"RerunIfEnvChanged":{"var":"RUST_LIBC_UNSTABLE_FREEBSD_VERSION","val":null}},{"RerunIfEnvChanged":{"var":"RUST_LIBC_UNSTABLE_MUSL_V1_2_3","val":null}},{"RerunIfEnvChanged":{"var":"RUST_LIBC_UNSTABLE_LINUX_TIME_BITS64","val":null}},{"RerunIfEnvChanged":{"var":"RUST_LIBC_UNSTABLE_GNU_FILE_OFFSET_BITS","val":null}},{"RerunIfEnvChanged":{"var":"RUST_LIBC_UNSTABLE_GNU_TIME_BITS","val":null}}],"rustflags":[],"config":0,"compile_kind":0}

View File

@@ -1 +0,0 @@
This file has an mtime of when this was started.

Some files were not shown because too many files have changed in this diff Show More