MyFSIO v0.2.0 Release #12
@@ -1,4 +1,3 @@
|
|||||||
"""Application factory for the mini S3-compatible object store."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
@@ -16,6 +15,7 @@ from flask_cors import CORS
|
|||||||
from flask_wtf.csrf import CSRFError
|
from flask_wtf.csrf import CSRFError
|
||||||
from werkzeug.middleware.proxy_fix import ProxyFix
|
from werkzeug.middleware.proxy_fix import ProxyFix
|
||||||
|
|
||||||
|
from .access_logging import AccessLoggingService
|
||||||
from .acl import AclService
|
from .acl import AclService
|
||||||
from .bucket_policies import BucketPolicyStore
|
from .bucket_policies import BucketPolicyStore
|
||||||
from .config import AppConfig
|
from .config import AppConfig
|
||||||
@@ -25,6 +25,8 @@ from .extensions import limiter, csrf
|
|||||||
from .iam import IamService
|
from .iam import IamService
|
||||||
from .kms import KMSManager
|
from .kms import KMSManager
|
||||||
from .lifecycle import LifecycleManager
|
from .lifecycle import LifecycleManager
|
||||||
|
from .notifications import NotificationService
|
||||||
|
from .object_lock import ObjectLockService
|
||||||
from .replication import ReplicationManager
|
from .replication import ReplicationManager
|
||||||
from .secret_store import EphemeralSecretStore
|
from .secret_store import EphemeralSecretStore
|
||||||
from .storage import ObjectStorage
|
from .storage import ObjectStorage
|
||||||
@@ -143,6 +145,10 @@ def create_app(
|
|||||||
storage = EncryptedObjectStorage(storage, encryption_manager)
|
storage = EncryptedObjectStorage(storage, encryption_manager)
|
||||||
|
|
||||||
acl_service = AclService(storage_root)
|
acl_service = AclService(storage_root)
|
||||||
|
object_lock_service = ObjectLockService(storage_root)
|
||||||
|
notification_service = NotificationService(storage_root)
|
||||||
|
access_logging_service = AccessLoggingService(storage_root)
|
||||||
|
access_logging_service.set_storage(storage)
|
||||||
|
|
||||||
lifecycle_manager = None
|
lifecycle_manager = None
|
||||||
if app.config.get("LIFECYCLE_ENABLED", False):
|
if app.config.get("LIFECYCLE_ENABLED", False):
|
||||||
@@ -164,6 +170,9 @@ def create_app(
|
|||||||
app.extensions["kms"] = kms_manager
|
app.extensions["kms"] = kms_manager
|
||||||
app.extensions["acl"] = acl_service
|
app.extensions["acl"] = acl_service
|
||||||
app.extensions["lifecycle"] = lifecycle_manager
|
app.extensions["lifecycle"] = lifecycle_manager
|
||||||
|
app.extensions["object_lock"] = object_lock_service
|
||||||
|
app.extensions["notifications"] = notification_service
|
||||||
|
app.extensions["access_logging"] = access_logging_service
|
||||||
|
|
||||||
@app.errorhandler(500)
|
@app.errorhandler(500)
|
||||||
def internal_error(error):
|
def internal_error(error):
|
||||||
|
|||||||
262
app/access_logging.py
Normal file
262
app/access_logging.py
Normal file
@@ -0,0 +1,262 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import queue
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class AccessLogEntry:
|
||||||
|
bucket_owner: str = "-"
|
||||||
|
bucket: str = "-"
|
||||||
|
timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
||||||
|
remote_ip: str = "-"
|
||||||
|
requester: str = "-"
|
||||||
|
request_id: str = field(default_factory=lambda: uuid.uuid4().hex[:16].upper())
|
||||||
|
operation: str = "-"
|
||||||
|
key: str = "-"
|
||||||
|
request_uri: str = "-"
|
||||||
|
http_status: int = 200
|
||||||
|
error_code: str = "-"
|
||||||
|
bytes_sent: int = 0
|
||||||
|
object_size: int = 0
|
||||||
|
total_time_ms: int = 0
|
||||||
|
turn_around_time_ms: int = 0
|
||||||
|
referrer: str = "-"
|
||||||
|
user_agent: str = "-"
|
||||||
|
version_id: str = "-"
|
||||||
|
host_id: str = "-"
|
||||||
|
signature_version: str = "SigV4"
|
||||||
|
cipher_suite: str = "-"
|
||||||
|
authentication_type: str = "AuthHeader"
|
||||||
|
host_header: str = "-"
|
||||||
|
tls_version: str = "-"
|
||||||
|
|
||||||
|
def to_log_line(self) -> str:
|
||||||
|
time_str = self.timestamp.strftime("[%d/%b/%Y:%H:%M:%S %z]")
|
||||||
|
return (
|
||||||
|
f'{self.bucket_owner} {self.bucket} {time_str} {self.remote_ip} '
|
||||||
|
f'{self.requester} {self.request_id} {self.operation} {self.key} '
|
||||||
|
f'"{self.request_uri}" {self.http_status} {self.error_code or "-"} '
|
||||||
|
f'{self.bytes_sent or "-"} {self.object_size or "-"} {self.total_time_ms or "-"} '
|
||||||
|
f'{self.turn_around_time_ms or "-"} "{self.referrer}" "{self.user_agent}" {self.version_id}'
|
||||||
|
)
|
||||||
|
|
||||||
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"bucket_owner": self.bucket_owner,
|
||||||
|
"bucket": self.bucket,
|
||||||
|
"timestamp": self.timestamp.isoformat(),
|
||||||
|
"remote_ip": self.remote_ip,
|
||||||
|
"requester": self.requester,
|
||||||
|
"request_id": self.request_id,
|
||||||
|
"operation": self.operation,
|
||||||
|
"key": self.key,
|
||||||
|
"request_uri": self.request_uri,
|
||||||
|
"http_status": self.http_status,
|
||||||
|
"error_code": self.error_code,
|
||||||
|
"bytes_sent": self.bytes_sent,
|
||||||
|
"object_size": self.object_size,
|
||||||
|
"total_time_ms": self.total_time_ms,
|
||||||
|
"referrer": self.referrer,
|
||||||
|
"user_agent": self.user_agent,
|
||||||
|
"version_id": self.version_id,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class LoggingConfiguration:
|
||||||
|
target_bucket: str
|
||||||
|
target_prefix: str = ""
|
||||||
|
enabled: bool = True
|
||||||
|
|
||||||
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"LoggingEnabled": {
|
||||||
|
"TargetBucket": self.target_bucket,
|
||||||
|
"TargetPrefix": self.target_prefix,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, data: Dict[str, Any]) -> Optional["LoggingConfiguration"]:
|
||||||
|
logging_enabled = data.get("LoggingEnabled")
|
||||||
|
if not logging_enabled:
|
||||||
|
return None
|
||||||
|
return cls(
|
||||||
|
target_bucket=logging_enabled.get("TargetBucket", ""),
|
||||||
|
target_prefix=logging_enabled.get("TargetPrefix", ""),
|
||||||
|
enabled=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class AccessLoggingService:
|
||||||
|
def __init__(self, storage_root: Path, flush_interval: int = 60, max_buffer_size: int = 1000):
|
||||||
|
self.storage_root = storage_root
|
||||||
|
self.flush_interval = flush_interval
|
||||||
|
self.max_buffer_size = max_buffer_size
|
||||||
|
self._configs: Dict[str, LoggingConfiguration] = {}
|
||||||
|
self._buffer: Dict[str, List[AccessLogEntry]] = {}
|
||||||
|
self._buffer_lock = threading.Lock()
|
||||||
|
self._shutdown = threading.Event()
|
||||||
|
self._storage = None
|
||||||
|
|
||||||
|
self._flush_thread = threading.Thread(target=self._flush_loop, name="access-log-flush", daemon=True)
|
||||||
|
self._flush_thread.start()
|
||||||
|
|
||||||
|
def set_storage(self, storage: Any) -> None:
|
||||||
|
self._storage = storage
|
||||||
|
|
||||||
|
def _config_path(self, bucket_name: str) -> Path:
|
||||||
|
return self.storage_root / ".myfsio.sys" / "buckets" / bucket_name / "logging.json"
|
||||||
|
|
||||||
|
def get_bucket_logging(self, bucket_name: str) -> Optional[LoggingConfiguration]:
|
||||||
|
if bucket_name in self._configs:
|
||||||
|
return self._configs[bucket_name]
|
||||||
|
|
||||||
|
config_path = self._config_path(bucket_name)
|
||||||
|
if not config_path.exists():
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
data = json.loads(config_path.read_text(encoding="utf-8"))
|
||||||
|
config = LoggingConfiguration.from_dict(data)
|
||||||
|
if config:
|
||||||
|
self._configs[bucket_name] = config
|
||||||
|
return config
|
||||||
|
except (json.JSONDecodeError, OSError) as e:
|
||||||
|
logger.warning(f"Failed to load logging config for {bucket_name}: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def set_bucket_logging(self, bucket_name: str, config: LoggingConfiguration) -> None:
|
||||||
|
config_path = self._config_path(bucket_name)
|
||||||
|
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
config_path.write_text(json.dumps(config.to_dict(), indent=2), encoding="utf-8")
|
||||||
|
self._configs[bucket_name] = config
|
||||||
|
|
||||||
|
def delete_bucket_logging(self, bucket_name: str) -> None:
|
||||||
|
config_path = self._config_path(bucket_name)
|
||||||
|
try:
|
||||||
|
if config_path.exists():
|
||||||
|
config_path.unlink()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
self._configs.pop(bucket_name, None)
|
||||||
|
|
||||||
|
def log_request(
|
||||||
|
self,
|
||||||
|
bucket_name: str,
|
||||||
|
*,
|
||||||
|
operation: str,
|
||||||
|
key: str = "-",
|
||||||
|
remote_ip: str = "-",
|
||||||
|
requester: str = "-",
|
||||||
|
request_uri: str = "-",
|
||||||
|
http_status: int = 200,
|
||||||
|
error_code: str = "",
|
||||||
|
bytes_sent: int = 0,
|
||||||
|
object_size: int = 0,
|
||||||
|
total_time_ms: int = 0,
|
||||||
|
referrer: str = "-",
|
||||||
|
user_agent: str = "-",
|
||||||
|
version_id: str = "-",
|
||||||
|
request_id: str = "",
|
||||||
|
) -> None:
|
||||||
|
config = self.get_bucket_logging(bucket_name)
|
||||||
|
if not config or not config.enabled:
|
||||||
|
return
|
||||||
|
|
||||||
|
entry = AccessLogEntry(
|
||||||
|
bucket_owner="local-owner",
|
||||||
|
bucket=bucket_name,
|
||||||
|
remote_ip=remote_ip,
|
||||||
|
requester=requester,
|
||||||
|
request_id=request_id or uuid.uuid4().hex[:16].upper(),
|
||||||
|
operation=operation,
|
||||||
|
key=key,
|
||||||
|
request_uri=request_uri,
|
||||||
|
http_status=http_status,
|
||||||
|
error_code=error_code,
|
||||||
|
bytes_sent=bytes_sent,
|
||||||
|
object_size=object_size,
|
||||||
|
total_time_ms=total_time_ms,
|
||||||
|
referrer=referrer,
|
||||||
|
user_agent=user_agent,
|
||||||
|
version_id=version_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
target_key = f"{config.target_bucket}:{config.target_prefix}"
|
||||||
|
with self._buffer_lock:
|
||||||
|
if target_key not in self._buffer:
|
||||||
|
self._buffer[target_key] = []
|
||||||
|
self._buffer[target_key].append(entry)
|
||||||
|
|
||||||
|
if len(self._buffer[target_key]) >= self.max_buffer_size:
|
||||||
|
self._flush_buffer(target_key)
|
||||||
|
|
||||||
|
def _flush_loop(self) -> None:
|
||||||
|
while not self._shutdown.is_set():
|
||||||
|
time.sleep(self.flush_interval)
|
||||||
|
self._flush_all()
|
||||||
|
|
||||||
|
def _flush_all(self) -> None:
|
||||||
|
with self._buffer_lock:
|
||||||
|
targets = list(self._buffer.keys())
|
||||||
|
|
||||||
|
for target_key in targets:
|
||||||
|
self._flush_buffer(target_key)
|
||||||
|
|
||||||
|
def _flush_buffer(self, target_key: str) -> None:
|
||||||
|
with self._buffer_lock:
|
||||||
|
entries = self._buffer.pop(target_key, [])
|
||||||
|
|
||||||
|
if not entries or not self._storage:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
bucket_name, prefix = target_key.split(":", 1)
|
||||||
|
except ValueError:
|
||||||
|
logger.error(f"Invalid target key: {target_key}")
|
||||||
|
return
|
||||||
|
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
log_key = f"{prefix}{now.strftime('%Y-%m-%d-%H-%M-%S')}-{uuid.uuid4().hex[:8]}"
|
||||||
|
|
||||||
|
log_content = "\n".join(entry.to_log_line() for entry in entries) + "\n"
|
||||||
|
|
||||||
|
try:
|
||||||
|
stream = io.BytesIO(log_content.encode("utf-8"))
|
||||||
|
self._storage.put_object(bucket_name, log_key, stream, enforce_quota=False)
|
||||||
|
logger.info(f"Flushed {len(entries)} access log entries to {bucket_name}/{log_key}")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to write access log to {bucket_name}/{log_key}: {e}")
|
||||||
|
with self._buffer_lock:
|
||||||
|
if target_key not in self._buffer:
|
||||||
|
self._buffer[target_key] = []
|
||||||
|
self._buffer[target_key] = entries + self._buffer[target_key]
|
||||||
|
|
||||||
|
def flush(self) -> None:
|
||||||
|
self._flush_all()
|
||||||
|
|
||||||
|
def shutdown(self) -> None:
|
||||||
|
self._shutdown.set()
|
||||||
|
self._flush_all()
|
||||||
|
self._flush_thread.join(timeout=5.0)
|
||||||
|
|
||||||
|
def get_stats(self) -> Dict[str, Any]:
|
||||||
|
with self._buffer_lock:
|
||||||
|
buffered = sum(len(entries) for entries in self._buffer.values())
|
||||||
|
return {
|
||||||
|
"buffered_entries": buffered,
|
||||||
|
"target_buckets": len(self._buffer),
|
||||||
|
}
|
||||||
@@ -1,4 +1,3 @@
|
|||||||
"""S3-compatible Access Control List (ACL) management."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
"""Bucket policy loader/enforcer with a subset of AWS semantics."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
"""Configuration helpers for the S3 clone application."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
"""Manage remote S3 connections."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
"""Encrypted storage layer that wraps ObjectStorage with encryption support."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import io
|
import io
|
||||||
|
|||||||
@@ -353,13 +353,113 @@ class EncryptionManager:
|
|||||||
return encryptor.decrypt_stream(stream, metadata)
|
return encryptor.decrypt_stream(stream, metadata)
|
||||||
|
|
||||||
|
|
||||||
|
class SSECEncryption(EncryptionProvider):
|
||||||
|
"""SSE-C: Server-Side Encryption with Customer-Provided Keys.
|
||||||
|
|
||||||
|
The client provides the encryption key with each request.
|
||||||
|
Server encrypts/decrypts but never stores the key.
|
||||||
|
|
||||||
|
Required headers for PUT:
|
||||||
|
- x-amz-server-side-encryption-customer-algorithm: AES256
|
||||||
|
- x-amz-server-side-encryption-customer-key: Base64-encoded 256-bit key
|
||||||
|
- x-amz-server-side-encryption-customer-key-MD5: Base64-encoded MD5 of key
|
||||||
|
"""
|
||||||
|
|
||||||
|
KEY_ID = "customer-provided"
|
||||||
|
|
||||||
|
def __init__(self, customer_key: bytes):
|
||||||
|
if len(customer_key) != 32:
|
||||||
|
raise EncryptionError("Customer key must be exactly 256 bits (32 bytes)")
|
||||||
|
self.customer_key = customer_key
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_headers(cls, headers: Dict[str, str]) -> "SSECEncryption":
|
||||||
|
algorithm = headers.get("x-amz-server-side-encryption-customer-algorithm", "")
|
||||||
|
if algorithm.upper() != "AES256":
|
||||||
|
raise EncryptionError(f"Unsupported SSE-C algorithm: {algorithm}. Only AES256 is supported.")
|
||||||
|
|
||||||
|
key_b64 = headers.get("x-amz-server-side-encryption-customer-key", "")
|
||||||
|
if not key_b64:
|
||||||
|
raise EncryptionError("Missing x-amz-server-side-encryption-customer-key header")
|
||||||
|
|
||||||
|
key_md5_b64 = headers.get("x-amz-server-side-encryption-customer-key-md5", "")
|
||||||
|
|
||||||
|
try:
|
||||||
|
customer_key = base64.b64decode(key_b64)
|
||||||
|
except Exception as e:
|
||||||
|
raise EncryptionError(f"Invalid base64 in customer key: {e}") from e
|
||||||
|
|
||||||
|
if len(customer_key) != 32:
|
||||||
|
raise EncryptionError(f"Customer key must be 256 bits, got {len(customer_key) * 8} bits")
|
||||||
|
|
||||||
|
if key_md5_b64:
|
||||||
|
import hashlib
|
||||||
|
expected_md5 = base64.b64encode(hashlib.md5(customer_key).digest()).decode()
|
||||||
|
if key_md5_b64 != expected_md5:
|
||||||
|
raise EncryptionError("Customer key MD5 mismatch")
|
||||||
|
|
||||||
|
return cls(customer_key)
|
||||||
|
|
||||||
|
def encrypt(self, plaintext: bytes, context: Dict[str, str] | None = None) -> EncryptionResult:
|
||||||
|
aesgcm = AESGCM(self.customer_key)
|
||||||
|
nonce = secrets.token_bytes(12)
|
||||||
|
ciphertext = aesgcm.encrypt(nonce, plaintext, None)
|
||||||
|
|
||||||
|
return EncryptionResult(
|
||||||
|
ciphertext=ciphertext,
|
||||||
|
nonce=nonce,
|
||||||
|
key_id=self.KEY_ID,
|
||||||
|
encrypted_data_key=b"",
|
||||||
|
)
|
||||||
|
|
||||||
|
def decrypt(self, ciphertext: bytes, nonce: bytes, encrypted_data_key: bytes,
|
||||||
|
key_id: str, context: Dict[str, str] | None = None) -> bytes:
|
||||||
|
aesgcm = AESGCM(self.customer_key)
|
||||||
|
try:
|
||||||
|
return aesgcm.decrypt(nonce, ciphertext, None)
|
||||||
|
except Exception as exc:
|
||||||
|
raise EncryptionError(f"SSE-C decryption failed: {exc}") from exc
|
||||||
|
|
||||||
|
def generate_data_key(self) -> tuple[bytes, bytes]:
|
||||||
|
return self.customer_key, b""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class SSECMetadata:
|
||||||
|
algorithm: str = "AES256"
|
||||||
|
nonce: bytes = b""
|
||||||
|
key_md5: str = ""
|
||||||
|
|
||||||
|
def to_dict(self) -> Dict[str, str]:
|
||||||
|
return {
|
||||||
|
"x-amz-server-side-encryption-customer-algorithm": self.algorithm,
|
||||||
|
"x-amz-encryption-nonce": base64.b64encode(self.nonce).decode(),
|
||||||
|
"x-amz-server-side-encryption-customer-key-MD5": self.key_md5,
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, data: Dict[str, str]) -> Optional["SSECMetadata"]:
|
||||||
|
algorithm = data.get("x-amz-server-side-encryption-customer-algorithm")
|
||||||
|
if not algorithm:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
nonce = base64.b64decode(data.get("x-amz-encryption-nonce", ""))
|
||||||
|
return cls(
|
||||||
|
algorithm=algorithm,
|
||||||
|
nonce=nonce,
|
||||||
|
key_md5=data.get("x-amz-server-side-encryption-customer-key-MD5", ""),
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
class ClientEncryptionHelper:
|
class ClientEncryptionHelper:
|
||||||
"""Helpers for client-side encryption.
|
"""Helpers for client-side encryption.
|
||||||
|
|
||||||
Client-side encryption is performed by the client, but this helper
|
Client-side encryption is performed by the client, but this helper
|
||||||
provides key generation and materials for clients that need them.
|
provides key generation and materials for clients that need them.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def generate_client_key() -> Dict[str, str]:
|
def generate_client_key() -> Dict[str, str]:
|
||||||
"""Generate a new client encryption key."""
|
"""Generate a new client encryption key."""
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
"""Standardized error handling for API and UI responses."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
"""Application-wide extension instances."""
|
|
||||||
from flask import g
|
from flask import g
|
||||||
from flask_limiter import Limiter
|
from flask_limiter import Limiter
|
||||||
from flask_limiter.util import get_remote_address
|
from flask_limiter.util import get_remote_address
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
"""Lightweight IAM-style user and policy management."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
"""Key Management Service (KMS) for encryption key management."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
"""KMS and encryption API endpoints."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
"""Lifecycle rule enforcement for S3-compatible storage."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|||||||
334
app/notifications.py
Normal file
334
app/notifications.py
Normal file
@@ -0,0 +1,334 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import queue
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class NotificationEvent:
|
||||||
|
event_name: str
|
||||||
|
bucket_name: str
|
||||||
|
object_key: str
|
||||||
|
object_size: int = 0
|
||||||
|
etag: str = ""
|
||||||
|
version_id: Optional[str] = None
|
||||||
|
timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
||||||
|
request_id: str = field(default_factory=lambda: uuid.uuid4().hex)
|
||||||
|
source_ip: str = ""
|
||||||
|
user_identity: str = ""
|
||||||
|
|
||||||
|
def to_s3_event(self) -> Dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"Records": [
|
||||||
|
{
|
||||||
|
"eventVersion": "2.1",
|
||||||
|
"eventSource": "myfsio:s3",
|
||||||
|
"awsRegion": "local",
|
||||||
|
"eventTime": self.timestamp.strftime("%Y-%m-%dT%H:%M:%S.000Z"),
|
||||||
|
"eventName": self.event_name,
|
||||||
|
"userIdentity": {
|
||||||
|
"principalId": self.user_identity or "ANONYMOUS",
|
||||||
|
},
|
||||||
|
"requestParameters": {
|
||||||
|
"sourceIPAddress": self.source_ip or "127.0.0.1",
|
||||||
|
},
|
||||||
|
"responseElements": {
|
||||||
|
"x-amz-request-id": self.request_id,
|
||||||
|
"x-amz-id-2": self.request_id,
|
||||||
|
},
|
||||||
|
"s3": {
|
||||||
|
"s3SchemaVersion": "1.0",
|
||||||
|
"configurationId": "notification",
|
||||||
|
"bucket": {
|
||||||
|
"name": self.bucket_name,
|
||||||
|
"ownerIdentity": {"principalId": "local"},
|
||||||
|
"arn": f"arn:aws:s3:::{self.bucket_name}",
|
||||||
|
},
|
||||||
|
"object": {
|
||||||
|
"key": self.object_key,
|
||||||
|
"size": self.object_size,
|
||||||
|
"eTag": self.etag,
|
||||||
|
"versionId": self.version_id or "null",
|
||||||
|
"sequencer": f"{int(time.time() * 1000):016X}",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class WebhookDestination:
|
||||||
|
url: str
|
||||||
|
headers: Dict[str, str] = field(default_factory=dict)
|
||||||
|
timeout_seconds: int = 30
|
||||||
|
retry_count: int = 3
|
||||||
|
retry_delay_seconds: int = 1
|
||||||
|
|
||||||
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"url": self.url,
|
||||||
|
"headers": self.headers,
|
||||||
|
"timeout_seconds": self.timeout_seconds,
|
||||||
|
"retry_count": self.retry_count,
|
||||||
|
"retry_delay_seconds": self.retry_delay_seconds,
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, data: Dict[str, Any]) -> "WebhookDestination":
|
||||||
|
return cls(
|
||||||
|
url=data.get("url", ""),
|
||||||
|
headers=data.get("headers", {}),
|
||||||
|
timeout_seconds=data.get("timeout_seconds", 30),
|
||||||
|
retry_count=data.get("retry_count", 3),
|
||||||
|
retry_delay_seconds=data.get("retry_delay_seconds", 1),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class NotificationConfiguration:
|
||||||
|
id: str
|
||||||
|
events: List[str]
|
||||||
|
destination: WebhookDestination
|
||||||
|
prefix_filter: str = ""
|
||||||
|
suffix_filter: str = ""
|
||||||
|
|
||||||
|
def matches_event(self, event_name: str, object_key: str) -> bool:
|
||||||
|
event_match = False
|
||||||
|
for pattern in self.events:
|
||||||
|
if pattern.endswith("*"):
|
||||||
|
base = pattern[:-1]
|
||||||
|
if event_name.startswith(base):
|
||||||
|
event_match = True
|
||||||
|
break
|
||||||
|
elif pattern == event_name:
|
||||||
|
event_match = True
|
||||||
|
break
|
||||||
|
|
||||||
|
if not event_match:
|
||||||
|
return False
|
||||||
|
|
||||||
|
if self.prefix_filter and not object_key.startswith(self.prefix_filter):
|
||||||
|
return False
|
||||||
|
if self.suffix_filter and not object_key.endswith(self.suffix_filter):
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"Id": self.id,
|
||||||
|
"Events": self.events,
|
||||||
|
"Destination": self.destination.to_dict(),
|
||||||
|
"Filter": {
|
||||||
|
"Key": {
|
||||||
|
"FilterRules": [
|
||||||
|
{"Name": "prefix", "Value": self.prefix_filter},
|
||||||
|
{"Name": "suffix", "Value": self.suffix_filter},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, data: Dict[str, Any]) -> "NotificationConfiguration":
|
||||||
|
prefix = ""
|
||||||
|
suffix = ""
|
||||||
|
filter_data = data.get("Filter", {})
|
||||||
|
key_filter = filter_data.get("Key", {})
|
||||||
|
for rule in key_filter.get("FilterRules", []):
|
||||||
|
if rule.get("Name") == "prefix":
|
||||||
|
prefix = rule.get("Value", "")
|
||||||
|
elif rule.get("Name") == "suffix":
|
||||||
|
suffix = rule.get("Value", "")
|
||||||
|
|
||||||
|
return cls(
|
||||||
|
id=data.get("Id", uuid.uuid4().hex),
|
||||||
|
events=data.get("Events", []),
|
||||||
|
destination=WebhookDestination.from_dict(data.get("Destination", {})),
|
||||||
|
prefix_filter=prefix,
|
||||||
|
suffix_filter=suffix,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class NotificationService:
|
||||||
|
def __init__(self, storage_root: Path, worker_count: int = 2):
|
||||||
|
self.storage_root = storage_root
|
||||||
|
self._configs: Dict[str, List[NotificationConfiguration]] = {}
|
||||||
|
self._queue: queue.Queue[tuple[NotificationEvent, WebhookDestination]] = queue.Queue()
|
||||||
|
self._workers: List[threading.Thread] = []
|
||||||
|
self._shutdown = threading.Event()
|
||||||
|
self._stats = {
|
||||||
|
"events_queued": 0,
|
||||||
|
"events_sent": 0,
|
||||||
|
"events_failed": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
for i in range(worker_count):
|
||||||
|
worker = threading.Thread(target=self._worker_loop, name=f"notification-worker-{i}", daemon=True)
|
||||||
|
worker.start()
|
||||||
|
self._workers.append(worker)
|
||||||
|
|
||||||
|
def _config_path(self, bucket_name: str) -> Path:
|
||||||
|
return self.storage_root / ".myfsio.sys" / "buckets" / bucket_name / "notifications.json"
|
||||||
|
|
||||||
|
def get_bucket_notifications(self, bucket_name: str) -> List[NotificationConfiguration]:
|
||||||
|
if bucket_name in self._configs:
|
||||||
|
return self._configs[bucket_name]
|
||||||
|
|
||||||
|
config_path = self._config_path(bucket_name)
|
||||||
|
if not config_path.exists():
|
||||||
|
return []
|
||||||
|
|
||||||
|
try:
|
||||||
|
data = json.loads(config_path.read_text(encoding="utf-8"))
|
||||||
|
configs = [NotificationConfiguration.from_dict(c) for c in data.get("configurations", [])]
|
||||||
|
self._configs[bucket_name] = configs
|
||||||
|
return configs
|
||||||
|
except (json.JSONDecodeError, OSError) as e:
|
||||||
|
logger.warning(f"Failed to load notification config for {bucket_name}: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
def set_bucket_notifications(
|
||||||
|
self, bucket_name: str, configurations: List[NotificationConfiguration]
|
||||||
|
) -> None:
|
||||||
|
config_path = self._config_path(bucket_name)
|
||||||
|
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
data = {"configurations": [c.to_dict() for c in configurations]}
|
||||||
|
config_path.write_text(json.dumps(data, indent=2), encoding="utf-8")
|
||||||
|
self._configs[bucket_name] = configurations
|
||||||
|
|
||||||
|
def delete_bucket_notifications(self, bucket_name: str) -> None:
|
||||||
|
config_path = self._config_path(bucket_name)
|
||||||
|
try:
|
||||||
|
if config_path.exists():
|
||||||
|
config_path.unlink()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
self._configs.pop(bucket_name, None)
|
||||||
|
|
||||||
|
def emit_event(self, event: NotificationEvent) -> None:
|
||||||
|
configurations = self.get_bucket_notifications(event.bucket_name)
|
||||||
|
if not configurations:
|
||||||
|
return
|
||||||
|
|
||||||
|
for config in configurations:
|
||||||
|
if config.matches_event(event.event_name, event.object_key):
|
||||||
|
self._queue.put((event, config.destination))
|
||||||
|
self._stats["events_queued"] += 1
|
||||||
|
logger.debug(
|
||||||
|
f"Queued notification for {event.event_name} on {event.bucket_name}/{event.object_key}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def emit_object_created(
|
||||||
|
self,
|
||||||
|
bucket_name: str,
|
||||||
|
object_key: str,
|
||||||
|
*,
|
||||||
|
size: int = 0,
|
||||||
|
etag: str = "",
|
||||||
|
version_id: Optional[str] = None,
|
||||||
|
request_id: str = "",
|
||||||
|
source_ip: str = "",
|
||||||
|
user_identity: str = "",
|
||||||
|
operation: str = "Put",
|
||||||
|
) -> None:
|
||||||
|
event = NotificationEvent(
|
||||||
|
event_name=f"s3:ObjectCreated:{operation}",
|
||||||
|
bucket_name=bucket_name,
|
||||||
|
object_key=object_key,
|
||||||
|
object_size=size,
|
||||||
|
etag=etag,
|
||||||
|
version_id=version_id,
|
||||||
|
request_id=request_id or uuid.uuid4().hex,
|
||||||
|
source_ip=source_ip,
|
||||||
|
user_identity=user_identity,
|
||||||
|
)
|
||||||
|
self.emit_event(event)
|
||||||
|
|
||||||
|
def emit_object_removed(
|
||||||
|
self,
|
||||||
|
bucket_name: str,
|
||||||
|
object_key: str,
|
||||||
|
*,
|
||||||
|
version_id: Optional[str] = None,
|
||||||
|
request_id: str = "",
|
||||||
|
source_ip: str = "",
|
||||||
|
user_identity: str = "",
|
||||||
|
operation: str = "Delete",
|
||||||
|
) -> None:
|
||||||
|
event = NotificationEvent(
|
||||||
|
event_name=f"s3:ObjectRemoved:{operation}",
|
||||||
|
bucket_name=bucket_name,
|
||||||
|
object_key=object_key,
|
||||||
|
version_id=version_id,
|
||||||
|
request_id=request_id or uuid.uuid4().hex,
|
||||||
|
source_ip=source_ip,
|
||||||
|
user_identity=user_identity,
|
||||||
|
)
|
||||||
|
self.emit_event(event)
|
||||||
|
|
||||||
|
def _worker_loop(self) -> None:
|
||||||
|
while not self._shutdown.is_set():
|
||||||
|
try:
|
||||||
|
event, destination = self._queue.get(timeout=1.0)
|
||||||
|
except queue.Empty:
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
self._send_notification(event, destination)
|
||||||
|
self._stats["events_sent"] += 1
|
||||||
|
except Exception as e:
|
||||||
|
self._stats["events_failed"] += 1
|
||||||
|
logger.error(f"Failed to send notification: {e}")
|
||||||
|
finally:
|
||||||
|
self._queue.task_done()
|
||||||
|
|
||||||
|
def _send_notification(self, event: NotificationEvent, destination: WebhookDestination) -> None:
|
||||||
|
payload = event.to_s3_event()
|
||||||
|
headers = {"Content-Type": "application/json", **destination.headers}
|
||||||
|
|
||||||
|
last_error = None
|
||||||
|
for attempt in range(destination.retry_count):
|
||||||
|
try:
|
||||||
|
response = requests.post(
|
||||||
|
destination.url,
|
||||||
|
json=payload,
|
||||||
|
headers=headers,
|
||||||
|
timeout=destination.timeout_seconds,
|
||||||
|
)
|
||||||
|
if response.status_code < 400:
|
||||||
|
logger.info(
|
||||||
|
f"Notification sent: {event.event_name} -> {destination.url} (status={response.status_code})"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
last_error = f"HTTP {response.status_code}: {response.text[:200]}"
|
||||||
|
except requests.RequestException as e:
|
||||||
|
last_error = str(e)
|
||||||
|
|
||||||
|
if attempt < destination.retry_count - 1:
|
||||||
|
time.sleep(destination.retry_delay_seconds * (attempt + 1))
|
||||||
|
|
||||||
|
raise RuntimeError(f"Failed after {destination.retry_count} attempts: {last_error}")
|
||||||
|
|
||||||
|
def get_stats(self) -> Dict[str, int]:
|
||||||
|
return dict(self._stats)
|
||||||
|
|
||||||
|
def shutdown(self) -> None:
|
||||||
|
self._shutdown.set()
|
||||||
|
for worker in self._workers:
|
||||||
|
worker.join(timeout=5.0)
|
||||||
234
app/object_lock.py
Normal file
234
app/object_lock.py
Normal file
@@ -0,0 +1,234 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from enum import Enum
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
|
|
||||||
|
class RetentionMode(Enum):
|
||||||
|
GOVERNANCE = "GOVERNANCE"
|
||||||
|
COMPLIANCE = "COMPLIANCE"
|
||||||
|
|
||||||
|
|
||||||
|
class ObjectLockError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ObjectLockRetention:
|
||||||
|
mode: RetentionMode
|
||||||
|
retain_until_date: datetime
|
||||||
|
|
||||||
|
def to_dict(self) -> Dict[str, str]:
|
||||||
|
return {
|
||||||
|
"Mode": self.mode.value,
|
||||||
|
"RetainUntilDate": self.retain_until_date.isoformat(),
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, data: Dict[str, Any]) -> Optional["ObjectLockRetention"]:
|
||||||
|
if not data:
|
||||||
|
return None
|
||||||
|
mode_str = data.get("Mode")
|
||||||
|
date_str = data.get("RetainUntilDate")
|
||||||
|
if not mode_str or not date_str:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
mode = RetentionMode(mode_str)
|
||||||
|
retain_until = datetime.fromisoformat(date_str.replace("Z", "+00:00"))
|
||||||
|
return cls(mode=mode, retain_until_date=retain_until)
|
||||||
|
except (ValueError, KeyError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def is_expired(self) -> bool:
|
||||||
|
return datetime.now(timezone.utc) > self.retain_until_date
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ObjectLockConfig:
|
||||||
|
enabled: bool = False
|
||||||
|
default_retention: Optional[ObjectLockRetention] = None
|
||||||
|
|
||||||
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
|
result: Dict[str, Any] = {"ObjectLockEnabled": "Enabled" if self.enabled else "Disabled"}
|
||||||
|
if self.default_retention:
|
||||||
|
result["Rule"] = {
|
||||||
|
"DefaultRetention": {
|
||||||
|
"Mode": self.default_retention.mode.value,
|
||||||
|
"Days": None,
|
||||||
|
"Years": None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, data: Dict[str, Any]) -> "ObjectLockConfig":
|
||||||
|
enabled = data.get("ObjectLockEnabled") == "Enabled"
|
||||||
|
default_retention = None
|
||||||
|
rule = data.get("Rule")
|
||||||
|
if rule and "DefaultRetention" in rule:
|
||||||
|
dr = rule["DefaultRetention"]
|
||||||
|
mode_str = dr.get("Mode", "GOVERNANCE")
|
||||||
|
days = dr.get("Days")
|
||||||
|
years = dr.get("Years")
|
||||||
|
if days or years:
|
||||||
|
from datetime import timedelta
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
if years:
|
||||||
|
delta = timedelta(days=int(years) * 365)
|
||||||
|
else:
|
||||||
|
delta = timedelta(days=int(days))
|
||||||
|
default_retention = ObjectLockRetention(
|
||||||
|
mode=RetentionMode(mode_str),
|
||||||
|
retain_until_date=now + delta,
|
||||||
|
)
|
||||||
|
return cls(enabled=enabled, default_retention=default_retention)
|
||||||
|
|
||||||
|
|
||||||
|
class ObjectLockService:
|
||||||
|
def __init__(self, storage_root: Path):
|
||||||
|
self.storage_root = storage_root
|
||||||
|
self._config_cache: Dict[str, ObjectLockConfig] = {}
|
||||||
|
|
||||||
|
def _bucket_lock_config_path(self, bucket_name: str) -> Path:
|
||||||
|
return self.storage_root / ".myfsio.sys" / "buckets" / bucket_name / "object_lock.json"
|
||||||
|
|
||||||
|
def _object_lock_meta_path(self, bucket_name: str, object_key: str) -> Path:
|
||||||
|
safe_key = object_key.replace("/", "_").replace("\\", "_")
|
||||||
|
return (
|
||||||
|
self.storage_root / ".myfsio.sys" / "buckets" / bucket_name /
|
||||||
|
"locks" / f"{safe_key}.lock.json"
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_bucket_lock_config(self, bucket_name: str) -> ObjectLockConfig:
|
||||||
|
if bucket_name in self._config_cache:
|
||||||
|
return self._config_cache[bucket_name]
|
||||||
|
|
||||||
|
config_path = self._bucket_lock_config_path(bucket_name)
|
||||||
|
if not config_path.exists():
|
||||||
|
return ObjectLockConfig(enabled=False)
|
||||||
|
|
||||||
|
try:
|
||||||
|
data = json.loads(config_path.read_text(encoding="utf-8"))
|
||||||
|
config = ObjectLockConfig.from_dict(data)
|
||||||
|
self._config_cache[bucket_name] = config
|
||||||
|
return config
|
||||||
|
except (json.JSONDecodeError, OSError):
|
||||||
|
return ObjectLockConfig(enabled=False)
|
||||||
|
|
||||||
|
def set_bucket_lock_config(self, bucket_name: str, config: ObjectLockConfig) -> None:
|
||||||
|
config_path = self._bucket_lock_config_path(bucket_name)
|
||||||
|
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
config_path.write_text(json.dumps(config.to_dict()), encoding="utf-8")
|
||||||
|
self._config_cache[bucket_name] = config
|
||||||
|
|
||||||
|
def enable_bucket_lock(self, bucket_name: str) -> None:
|
||||||
|
config = self.get_bucket_lock_config(bucket_name)
|
||||||
|
config.enabled = True
|
||||||
|
self.set_bucket_lock_config(bucket_name, config)
|
||||||
|
|
||||||
|
def is_bucket_lock_enabled(self, bucket_name: str) -> bool:
|
||||||
|
return self.get_bucket_lock_config(bucket_name).enabled
|
||||||
|
|
||||||
|
def get_object_retention(self, bucket_name: str, object_key: str) -> Optional[ObjectLockRetention]:
|
||||||
|
meta_path = self._object_lock_meta_path(bucket_name, object_key)
|
||||||
|
if not meta_path.exists():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
data = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||||
|
return ObjectLockRetention.from_dict(data.get("retention", {}))
|
||||||
|
except (json.JSONDecodeError, OSError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def set_object_retention(
|
||||||
|
self,
|
||||||
|
bucket_name: str,
|
||||||
|
object_key: str,
|
||||||
|
retention: ObjectLockRetention,
|
||||||
|
bypass_governance: bool = False,
|
||||||
|
) -> None:
|
||||||
|
existing = self.get_object_retention(bucket_name, object_key)
|
||||||
|
if existing and not existing.is_expired():
|
||||||
|
if existing.mode == RetentionMode.COMPLIANCE:
|
||||||
|
raise ObjectLockError(
|
||||||
|
"Cannot modify retention on object with COMPLIANCE mode until retention expires"
|
||||||
|
)
|
||||||
|
if existing.mode == RetentionMode.GOVERNANCE and not bypass_governance:
|
||||||
|
raise ObjectLockError(
|
||||||
|
"Cannot modify GOVERNANCE retention without bypass-governance permission"
|
||||||
|
)
|
||||||
|
|
||||||
|
meta_path = self._object_lock_meta_path(bucket_name, object_key)
|
||||||
|
meta_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
existing_data: Dict[str, Any] = {}
|
||||||
|
if meta_path.exists():
|
||||||
|
try:
|
||||||
|
existing_data = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||||
|
except (json.JSONDecodeError, OSError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
existing_data["retention"] = retention.to_dict()
|
||||||
|
meta_path.write_text(json.dumps(existing_data), encoding="utf-8")
|
||||||
|
|
||||||
|
def get_legal_hold(self, bucket_name: str, object_key: str) -> bool:
|
||||||
|
meta_path = self._object_lock_meta_path(bucket_name, object_key)
|
||||||
|
if not meta_path.exists():
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
data = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||||
|
return data.get("legal_hold", False)
|
||||||
|
except (json.JSONDecodeError, OSError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def set_legal_hold(self, bucket_name: str, object_key: str, enabled: bool) -> None:
|
||||||
|
meta_path = self._object_lock_meta_path(bucket_name, object_key)
|
||||||
|
meta_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
existing_data: Dict[str, Any] = {}
|
||||||
|
if meta_path.exists():
|
||||||
|
try:
|
||||||
|
existing_data = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||||
|
except (json.JSONDecodeError, OSError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
existing_data["legal_hold"] = enabled
|
||||||
|
meta_path.write_text(json.dumps(existing_data), encoding="utf-8")
|
||||||
|
|
||||||
|
def can_delete_object(
|
||||||
|
self,
|
||||||
|
bucket_name: str,
|
||||||
|
object_key: str,
|
||||||
|
bypass_governance: bool = False,
|
||||||
|
) -> tuple[bool, str]:
|
||||||
|
if self.get_legal_hold(bucket_name, object_key):
|
||||||
|
return False, "Object is under legal hold"
|
||||||
|
|
||||||
|
retention = self.get_object_retention(bucket_name, object_key)
|
||||||
|
if retention and not retention.is_expired():
|
||||||
|
if retention.mode == RetentionMode.COMPLIANCE:
|
||||||
|
return False, f"Object is locked in COMPLIANCE mode until {retention.retain_until_date.isoformat()}"
|
||||||
|
if retention.mode == RetentionMode.GOVERNANCE:
|
||||||
|
if not bypass_governance:
|
||||||
|
return False, f"Object is locked in GOVERNANCE mode until {retention.retain_until_date.isoformat()}"
|
||||||
|
|
||||||
|
return True, ""
|
||||||
|
|
||||||
|
def can_overwrite_object(
|
||||||
|
self,
|
||||||
|
bucket_name: str,
|
||||||
|
object_key: str,
|
||||||
|
bypass_governance: bool = False,
|
||||||
|
) -> tuple[bool, str]:
|
||||||
|
return self.can_delete_object(bucket_name, object_key, bypass_governance)
|
||||||
|
|
||||||
|
def delete_object_lock_metadata(self, bucket_name: str, object_key: str) -> None:
|
||||||
|
meta_path = self._object_lock_meta_path(bucket_name, object_key)
|
||||||
|
try:
|
||||||
|
if meta_path.exists():
|
||||||
|
meta_path.unlink()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
@@ -1,4 +1,3 @@
|
|||||||
"""Background replication worker."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
@@ -32,13 +31,9 @@ REPLICATION_MODE_ALL = "all"
|
|||||||
|
|
||||||
def _create_s3_client(connection: RemoteConnection, *, health_check: bool = False) -> Any:
|
def _create_s3_client(connection: RemoteConnection, *, health_check: bool = False) -> Any:
|
||||||
"""Create a boto3 S3 client for the given connection.
|
"""Create a boto3 S3 client for the given connection.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
connection: Remote S3 connection configuration
|
connection: Remote S3 connection configuration
|
||||||
health_check: If True, use minimal retries for quick health checks
|
health_check: If True, use minimal retries for quick health checks
|
||||||
|
|
||||||
Returns:
|
|
||||||
Configured boto3 S3 client
|
|
||||||
"""
|
"""
|
||||||
config = Config(
|
config = Config(
|
||||||
user_agent_extra=REPLICATION_USER_AGENT,
|
user_agent_extra=REPLICATION_USER_AGENT,
|
||||||
|
|||||||
405
app/s3_api.py
405
app/s3_api.py
@@ -16,10 +16,14 @@ from xml.etree.ElementTree import Element, SubElement, tostring, fromstring, Par
|
|||||||
from flask import Blueprint, Response, current_app, jsonify, request, g
|
from flask import Blueprint, Response, current_app, jsonify, request, g
|
||||||
from werkzeug.http import http_date
|
from werkzeug.http import http_date
|
||||||
|
|
||||||
|
from .access_logging import AccessLoggingService, LoggingConfiguration
|
||||||
from .acl import AclService
|
from .acl import AclService
|
||||||
from .bucket_policies import BucketPolicyStore
|
from .bucket_policies import BucketPolicyStore
|
||||||
|
from .encryption import SSECEncryption, SSECMetadata, EncryptionError
|
||||||
from .extensions import limiter
|
from .extensions import limiter
|
||||||
from .iam import IamError, Principal
|
from .iam import IamError, Principal
|
||||||
|
from .notifications import NotificationService, NotificationConfiguration, WebhookDestination
|
||||||
|
from .object_lock import ObjectLockService, ObjectLockRetention, ObjectLockConfig, ObjectLockError, RetentionMode
|
||||||
from .replication import ReplicationManager
|
from .replication import ReplicationManager
|
||||||
from .storage import ObjectStorage, StorageError, QuotaExceededError
|
from .storage import ObjectStorage, StorageError, QuotaExceededError
|
||||||
|
|
||||||
@@ -49,6 +53,18 @@ def _bucket_policies() -> BucketPolicyStore:
|
|||||||
return store
|
return store
|
||||||
|
|
||||||
|
|
||||||
|
def _object_lock() -> ObjectLockService:
|
||||||
|
return current_app.extensions["object_lock"]
|
||||||
|
|
||||||
|
|
||||||
|
def _notifications() -> NotificationService:
|
||||||
|
return current_app.extensions["notifications"]
|
||||||
|
|
||||||
|
|
||||||
|
def _access_logging() -> AccessLoggingService:
|
||||||
|
return current_app.extensions["access_logging"]
|
||||||
|
|
||||||
|
|
||||||
def _xml_response(element: Element, status: int = 200) -> Response:
|
def _xml_response(element: Element, status: int = 200) -> Response:
|
||||||
xml_bytes = tostring(element, encoding="utf-8")
|
xml_bytes = tostring(element, encoding="utf-8")
|
||||||
return Response(xml_bytes, status=status, mimetype="application/xml")
|
return Response(xml_bytes, status=status, mimetype="application/xml")
|
||||||
@@ -897,6 +913,9 @@ def _maybe_handle_bucket_subresource(bucket_name: str) -> Response | None:
|
|||||||
"versions": _bucket_list_versions_handler,
|
"versions": _bucket_list_versions_handler,
|
||||||
"lifecycle": _bucket_lifecycle_handler,
|
"lifecycle": _bucket_lifecycle_handler,
|
||||||
"quota": _bucket_quota_handler,
|
"quota": _bucket_quota_handler,
|
||||||
|
"object-lock": _bucket_object_lock_handler,
|
||||||
|
"notification": _bucket_notification_handler,
|
||||||
|
"logging": _bucket_logging_handler,
|
||||||
}
|
}
|
||||||
requested = [key for key in handlers if key in request.args]
|
requested = [key for key in handlers if key in request.args]
|
||||||
if not requested:
|
if not requested:
|
||||||
@@ -1567,6 +1586,336 @@ def _bucket_quota_handler(bucket_name: str) -> Response:
|
|||||||
return Response(status=204)
|
return Response(status=204)
|
||||||
|
|
||||||
|
|
||||||
|
def _bucket_object_lock_handler(bucket_name: str) -> Response:
|
||||||
|
if request.method not in {"GET", "PUT"}:
|
||||||
|
return _method_not_allowed(["GET", "PUT"])
|
||||||
|
|
||||||
|
principal, error = _require_principal()
|
||||||
|
if error:
|
||||||
|
return error
|
||||||
|
try:
|
||||||
|
_authorize_action(principal, bucket_name, "policy")
|
||||||
|
except IamError as exc:
|
||||||
|
return _error_response("AccessDenied", str(exc), 403)
|
||||||
|
|
||||||
|
storage = _storage()
|
||||||
|
if not storage.bucket_exists(bucket_name):
|
||||||
|
return _error_response("NoSuchBucket", "Bucket does not exist", 404)
|
||||||
|
|
||||||
|
lock_service = _object_lock()
|
||||||
|
|
||||||
|
if request.method == "GET":
|
||||||
|
config = lock_service.get_bucket_lock_config(bucket_name)
|
||||||
|
root = Element("ObjectLockConfiguration", xmlns="http://s3.amazonaws.com/doc/2006-03-01/")
|
||||||
|
SubElement(root, "ObjectLockEnabled").text = "Enabled" if config.enabled else "Disabled"
|
||||||
|
return _xml_response(root)
|
||||||
|
|
||||||
|
payload = request.get_data(cache=False) or b""
|
||||||
|
if not payload.strip():
|
||||||
|
return _error_response("MalformedXML", "Request body is required", 400)
|
||||||
|
|
||||||
|
try:
|
||||||
|
root = fromstring(payload)
|
||||||
|
except ParseError:
|
||||||
|
return _error_response("MalformedXML", "Unable to parse XML document", 400)
|
||||||
|
|
||||||
|
enabled_el = root.find("{*}ObjectLockEnabled") or root.find("ObjectLockEnabled")
|
||||||
|
enabled = (enabled_el.text or "").strip() == "Enabled" if enabled_el is not None else False
|
||||||
|
|
||||||
|
config = ObjectLockConfig(enabled=enabled)
|
||||||
|
lock_service.set_bucket_lock_config(bucket_name, config)
|
||||||
|
|
||||||
|
current_app.logger.info("Bucket object lock updated", extra={"bucket": bucket_name, "enabled": enabled})
|
||||||
|
return Response(status=200)
|
||||||
|
|
||||||
|
|
||||||
|
def _bucket_notification_handler(bucket_name: str) -> Response:
|
||||||
|
if request.method not in {"GET", "PUT", "DELETE"}:
|
||||||
|
return _method_not_allowed(["GET", "PUT", "DELETE"])
|
||||||
|
|
||||||
|
principal, error = _require_principal()
|
||||||
|
if error:
|
||||||
|
return error
|
||||||
|
try:
|
||||||
|
_authorize_action(principal, bucket_name, "policy")
|
||||||
|
except IamError as exc:
|
||||||
|
return _error_response("AccessDenied", str(exc), 403)
|
||||||
|
|
||||||
|
storage = _storage()
|
||||||
|
if not storage.bucket_exists(bucket_name):
|
||||||
|
return _error_response("NoSuchBucket", "Bucket does not exist", 404)
|
||||||
|
|
||||||
|
notification_service = _notifications()
|
||||||
|
|
||||||
|
if request.method == "GET":
|
||||||
|
configs = notification_service.get_bucket_notifications(bucket_name)
|
||||||
|
root = Element("NotificationConfiguration", xmlns="http://s3.amazonaws.com/doc/2006-03-01/")
|
||||||
|
for config in configs:
|
||||||
|
webhook_el = SubElement(root, "WebhookConfiguration")
|
||||||
|
SubElement(webhook_el, "Id").text = config.id
|
||||||
|
for event in config.events:
|
||||||
|
SubElement(webhook_el, "Event").text = event
|
||||||
|
dest_el = SubElement(webhook_el, "Destination")
|
||||||
|
SubElement(dest_el, "Url").text = config.destination.url
|
||||||
|
if config.prefix_filter or config.suffix_filter:
|
||||||
|
filter_el = SubElement(webhook_el, "Filter")
|
||||||
|
key_el = SubElement(filter_el, "S3Key")
|
||||||
|
if config.prefix_filter:
|
||||||
|
rule_el = SubElement(key_el, "FilterRule")
|
||||||
|
SubElement(rule_el, "Name").text = "prefix"
|
||||||
|
SubElement(rule_el, "Value").text = config.prefix_filter
|
||||||
|
if config.suffix_filter:
|
||||||
|
rule_el = SubElement(key_el, "FilterRule")
|
||||||
|
SubElement(rule_el, "Name").text = "suffix"
|
||||||
|
SubElement(rule_el, "Value").text = config.suffix_filter
|
||||||
|
return _xml_response(root)
|
||||||
|
|
||||||
|
if request.method == "DELETE":
|
||||||
|
notification_service.delete_bucket_notifications(bucket_name)
|
||||||
|
current_app.logger.info("Bucket notifications deleted", extra={"bucket": bucket_name})
|
||||||
|
return Response(status=204)
|
||||||
|
|
||||||
|
payload = request.get_data(cache=False) or b""
|
||||||
|
if not payload.strip():
|
||||||
|
notification_service.delete_bucket_notifications(bucket_name)
|
||||||
|
return Response(status=200)
|
||||||
|
|
||||||
|
try:
|
||||||
|
root = fromstring(payload)
|
||||||
|
except ParseError:
|
||||||
|
return _error_response("MalformedXML", "Unable to parse XML document", 400)
|
||||||
|
|
||||||
|
configs: list[NotificationConfiguration] = []
|
||||||
|
for webhook_el in root.findall("{*}WebhookConfiguration") or root.findall("WebhookConfiguration"):
|
||||||
|
config_id = _find_element_text(webhook_el, "Id") or uuid.uuid4().hex
|
||||||
|
events = [el.text for el in webhook_el.findall("{*}Event") or webhook_el.findall("Event") if el.text]
|
||||||
|
|
||||||
|
dest_el = _find_element(webhook_el, "Destination")
|
||||||
|
url = _find_element_text(dest_el, "Url") if dest_el else ""
|
||||||
|
if not url:
|
||||||
|
return _error_response("InvalidArgument", "Destination URL is required", 400)
|
||||||
|
|
||||||
|
prefix = ""
|
||||||
|
suffix = ""
|
||||||
|
filter_el = _find_element(webhook_el, "Filter")
|
||||||
|
if filter_el:
|
||||||
|
key_el = _find_element(filter_el, "S3Key")
|
||||||
|
if key_el:
|
||||||
|
for rule_el in key_el.findall("{*}FilterRule") or key_el.findall("FilterRule"):
|
||||||
|
name = _find_element_text(rule_el, "Name")
|
||||||
|
value = _find_element_text(rule_el, "Value")
|
||||||
|
if name == "prefix":
|
||||||
|
prefix = value
|
||||||
|
elif name == "suffix":
|
||||||
|
suffix = value
|
||||||
|
|
||||||
|
configs.append(NotificationConfiguration(
|
||||||
|
id=config_id,
|
||||||
|
events=events,
|
||||||
|
destination=WebhookDestination(url=url),
|
||||||
|
prefix_filter=prefix,
|
||||||
|
suffix_filter=suffix,
|
||||||
|
))
|
||||||
|
|
||||||
|
notification_service.set_bucket_notifications(bucket_name, configs)
|
||||||
|
current_app.logger.info("Bucket notifications updated", extra={"bucket": bucket_name, "configs": len(configs)})
|
||||||
|
return Response(status=200)
|
||||||
|
|
||||||
|
|
||||||
|
def _bucket_logging_handler(bucket_name: str) -> Response:
|
||||||
|
if request.method not in {"GET", "PUT", "DELETE"}:
|
||||||
|
return _method_not_allowed(["GET", "PUT", "DELETE"])
|
||||||
|
|
||||||
|
principal, error = _require_principal()
|
||||||
|
if error:
|
||||||
|
return error
|
||||||
|
try:
|
||||||
|
_authorize_action(principal, bucket_name, "policy")
|
||||||
|
except IamError as exc:
|
||||||
|
return _error_response("AccessDenied", str(exc), 403)
|
||||||
|
|
||||||
|
storage = _storage()
|
||||||
|
if not storage.bucket_exists(bucket_name):
|
||||||
|
return _error_response("NoSuchBucket", "Bucket does not exist", 404)
|
||||||
|
|
||||||
|
logging_service = _access_logging()
|
||||||
|
|
||||||
|
if request.method == "GET":
|
||||||
|
config = logging_service.get_bucket_logging(bucket_name)
|
||||||
|
root = Element("BucketLoggingStatus", xmlns="http://s3.amazonaws.com/doc/2006-03-01/")
|
||||||
|
if config and config.enabled:
|
||||||
|
logging_enabled = SubElement(root, "LoggingEnabled")
|
||||||
|
SubElement(logging_enabled, "TargetBucket").text = config.target_bucket
|
||||||
|
SubElement(logging_enabled, "TargetPrefix").text = config.target_prefix
|
||||||
|
return _xml_response(root)
|
||||||
|
|
||||||
|
if request.method == "DELETE":
|
||||||
|
logging_service.delete_bucket_logging(bucket_name)
|
||||||
|
current_app.logger.info("Bucket logging deleted", extra={"bucket": bucket_name})
|
||||||
|
return Response(status=204)
|
||||||
|
|
||||||
|
payload = request.get_data(cache=False) or b""
|
||||||
|
if not payload.strip():
|
||||||
|
logging_service.delete_bucket_logging(bucket_name)
|
||||||
|
return Response(status=200)
|
||||||
|
|
||||||
|
try:
|
||||||
|
root = fromstring(payload)
|
||||||
|
except ParseError:
|
||||||
|
return _error_response("MalformedXML", "Unable to parse XML document", 400)
|
||||||
|
|
||||||
|
logging_enabled = _find_element(root, "LoggingEnabled")
|
||||||
|
if logging_enabled is None:
|
||||||
|
logging_service.delete_bucket_logging(bucket_name)
|
||||||
|
return Response(status=200)
|
||||||
|
|
||||||
|
target_bucket = _find_element_text(logging_enabled, "TargetBucket")
|
||||||
|
if not target_bucket:
|
||||||
|
return _error_response("InvalidArgument", "TargetBucket is required", 400)
|
||||||
|
|
||||||
|
if not storage.bucket_exists(target_bucket):
|
||||||
|
return _error_response("InvalidTargetBucketForLogging", "Target bucket does not exist", 400)
|
||||||
|
|
||||||
|
target_prefix = _find_element_text(logging_enabled, "TargetPrefix")
|
||||||
|
|
||||||
|
config = LoggingConfiguration(
|
||||||
|
target_bucket=target_bucket,
|
||||||
|
target_prefix=target_prefix,
|
||||||
|
enabled=True,
|
||||||
|
)
|
||||||
|
logging_service.set_bucket_logging(bucket_name, config)
|
||||||
|
|
||||||
|
current_app.logger.info(
|
||||||
|
"Bucket logging updated",
|
||||||
|
extra={"bucket": bucket_name, "target_bucket": target_bucket, "target_prefix": target_prefix}
|
||||||
|
)
|
||||||
|
return Response(status=200)
|
||||||
|
|
||||||
|
|
||||||
|
def _object_retention_handler(bucket_name: str, object_key: str) -> Response:
|
||||||
|
if request.method not in {"GET", "PUT"}:
|
||||||
|
return _method_not_allowed(["GET", "PUT"])
|
||||||
|
|
||||||
|
principal, error = _require_principal()
|
||||||
|
if error:
|
||||||
|
return error
|
||||||
|
try:
|
||||||
|
_authorize_action(principal, bucket_name, "write" if request.method == "PUT" else "read", object_key=object_key)
|
||||||
|
except IamError as exc:
|
||||||
|
return _error_response("AccessDenied", str(exc), 403)
|
||||||
|
|
||||||
|
storage = _storage()
|
||||||
|
if not storage.bucket_exists(bucket_name):
|
||||||
|
return _error_response("NoSuchBucket", "Bucket does not exist", 404)
|
||||||
|
|
||||||
|
try:
|
||||||
|
storage.get_object_path(bucket_name, object_key)
|
||||||
|
except StorageError:
|
||||||
|
return _error_response("NoSuchKey", "Object does not exist", 404)
|
||||||
|
|
||||||
|
lock_service = _object_lock()
|
||||||
|
|
||||||
|
if request.method == "GET":
|
||||||
|
retention = lock_service.get_object_retention(bucket_name, object_key)
|
||||||
|
if not retention:
|
||||||
|
return _error_response("NoSuchObjectLockConfiguration", "No retention policy", 404)
|
||||||
|
|
||||||
|
root = Element("Retention", xmlns="http://s3.amazonaws.com/doc/2006-03-01/")
|
||||||
|
SubElement(root, "Mode").text = retention.mode.value
|
||||||
|
SubElement(root, "RetainUntilDate").text = retention.retain_until_date.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
||||||
|
return _xml_response(root)
|
||||||
|
|
||||||
|
payload = request.get_data(cache=False) or b""
|
||||||
|
if not payload.strip():
|
||||||
|
return _error_response("MalformedXML", "Request body is required", 400)
|
||||||
|
|
||||||
|
try:
|
||||||
|
root = fromstring(payload)
|
||||||
|
except ParseError:
|
||||||
|
return _error_response("MalformedXML", "Unable to parse XML document", 400)
|
||||||
|
|
||||||
|
mode_str = _find_element_text(root, "Mode")
|
||||||
|
retain_until_str = _find_element_text(root, "RetainUntilDate")
|
||||||
|
|
||||||
|
if not mode_str or not retain_until_str:
|
||||||
|
return _error_response("InvalidArgument", "Mode and RetainUntilDate are required", 400)
|
||||||
|
|
||||||
|
try:
|
||||||
|
mode = RetentionMode(mode_str)
|
||||||
|
except ValueError:
|
||||||
|
return _error_response("InvalidArgument", f"Invalid retention mode: {mode_str}", 400)
|
||||||
|
|
||||||
|
try:
|
||||||
|
retain_until = datetime.fromisoformat(retain_until_str.replace("Z", "+00:00"))
|
||||||
|
except ValueError:
|
||||||
|
return _error_response("InvalidArgument", f"Invalid date format: {retain_until_str}", 400)
|
||||||
|
|
||||||
|
bypass = request.headers.get("x-amz-bypass-governance-retention", "").lower() == "true"
|
||||||
|
|
||||||
|
retention = ObjectLockRetention(mode=mode, retain_until_date=retain_until)
|
||||||
|
try:
|
||||||
|
lock_service.set_object_retention(bucket_name, object_key, retention, bypass_governance=bypass)
|
||||||
|
except ObjectLockError as exc:
|
||||||
|
return _error_response("AccessDenied", str(exc), 403)
|
||||||
|
|
||||||
|
current_app.logger.info(
|
||||||
|
"Object retention set",
|
||||||
|
extra={"bucket": bucket_name, "key": object_key, "mode": mode_str, "until": retain_until_str}
|
||||||
|
)
|
||||||
|
return Response(status=200)
|
||||||
|
|
||||||
|
|
||||||
|
def _object_legal_hold_handler(bucket_name: str, object_key: str) -> Response:
|
||||||
|
if request.method not in {"GET", "PUT"}:
|
||||||
|
return _method_not_allowed(["GET", "PUT"])
|
||||||
|
|
||||||
|
principal, error = _require_principal()
|
||||||
|
if error:
|
||||||
|
return error
|
||||||
|
try:
|
||||||
|
_authorize_action(principal, bucket_name, "write" if request.method == "PUT" else "read", object_key=object_key)
|
||||||
|
except IamError as exc:
|
||||||
|
return _error_response("AccessDenied", str(exc), 403)
|
||||||
|
|
||||||
|
storage = _storage()
|
||||||
|
if not storage.bucket_exists(bucket_name):
|
||||||
|
return _error_response("NoSuchBucket", "Bucket does not exist", 404)
|
||||||
|
|
||||||
|
try:
|
||||||
|
storage.get_object_path(bucket_name, object_key)
|
||||||
|
except StorageError:
|
||||||
|
return _error_response("NoSuchKey", "Object does not exist", 404)
|
||||||
|
|
||||||
|
lock_service = _object_lock()
|
||||||
|
|
||||||
|
if request.method == "GET":
|
||||||
|
enabled = lock_service.get_legal_hold(bucket_name, object_key)
|
||||||
|
root = Element("LegalHold", xmlns="http://s3.amazonaws.com/doc/2006-03-01/")
|
||||||
|
SubElement(root, "Status").text = "ON" if enabled else "OFF"
|
||||||
|
return _xml_response(root)
|
||||||
|
|
||||||
|
payload = request.get_data(cache=False) or b""
|
||||||
|
if not payload.strip():
|
||||||
|
return _error_response("MalformedXML", "Request body is required", 400)
|
||||||
|
|
||||||
|
try:
|
||||||
|
root = fromstring(payload)
|
||||||
|
except ParseError:
|
||||||
|
return _error_response("MalformedXML", "Unable to parse XML document", 400)
|
||||||
|
|
||||||
|
status = _find_element_text(root, "Status")
|
||||||
|
if status not in {"ON", "OFF"}:
|
||||||
|
return _error_response("InvalidArgument", "Status must be ON or OFF", 400)
|
||||||
|
|
||||||
|
lock_service.set_legal_hold(bucket_name, object_key, status == "ON")
|
||||||
|
|
||||||
|
current_app.logger.info(
|
||||||
|
"Object legal hold set",
|
||||||
|
extra={"bucket": bucket_name, "key": object_key, "status": status}
|
||||||
|
)
|
||||||
|
return Response(status=200)
|
||||||
|
|
||||||
|
|
||||||
def _bulk_delete_handler(bucket_name: str) -> Response:
|
def _bulk_delete_handler(bucket_name: str) -> Response:
|
||||||
principal, error = _require_principal()
|
principal, error = _require_principal()
|
||||||
if error:
|
if error:
|
||||||
@@ -1871,6 +2220,12 @@ def object_handler(bucket_name: str, object_key: str):
|
|||||||
if "tagging" in request.args:
|
if "tagging" in request.args:
|
||||||
return _object_tagging_handler(bucket_name, object_key)
|
return _object_tagging_handler(bucket_name, object_key)
|
||||||
|
|
||||||
|
if "retention" in request.args:
|
||||||
|
return _object_retention_handler(bucket_name, object_key)
|
||||||
|
|
||||||
|
if "legal-hold" in request.args:
|
||||||
|
return _object_legal_hold_handler(bucket_name, object_key)
|
||||||
|
|
||||||
if request.method == "POST":
|
if request.method == "POST":
|
||||||
if "uploads" in request.args:
|
if "uploads" in request.args:
|
||||||
return _initiate_multipart_upload(bucket_name, object_key)
|
return _initiate_multipart_upload(bucket_name, object_key)
|
||||||
@@ -1886,22 +2241,28 @@ def object_handler(bucket_name: str, object_key: str):
|
|||||||
if copy_source:
|
if copy_source:
|
||||||
return _copy_object(bucket_name, object_key, copy_source)
|
return _copy_object(bucket_name, object_key, copy_source)
|
||||||
|
|
||||||
_, error = _object_principal("write", bucket_name, object_key)
|
principal, error = _object_principal("write", bucket_name, object_key)
|
||||||
if error:
|
if error:
|
||||||
return error
|
return error
|
||||||
|
|
||||||
|
bypass_governance = request.headers.get("x-amz-bypass-governance-retention", "").lower() == "true"
|
||||||
|
lock_service = _object_lock()
|
||||||
|
can_overwrite, lock_reason = lock_service.can_overwrite_object(bucket_name, object_key, bypass_governance=bypass_governance)
|
||||||
|
if not can_overwrite:
|
||||||
|
return _error_response("AccessDenied", lock_reason, 403)
|
||||||
|
|
||||||
stream = request.stream
|
stream = request.stream
|
||||||
content_encoding = request.headers.get("Content-Encoding", "").lower()
|
content_encoding = request.headers.get("Content-Encoding", "").lower()
|
||||||
if "aws-chunked" in content_encoding:
|
if "aws-chunked" in content_encoding:
|
||||||
stream = AwsChunkedDecoder(stream)
|
stream = AwsChunkedDecoder(stream)
|
||||||
|
|
||||||
metadata = _extract_request_metadata()
|
metadata = _extract_request_metadata()
|
||||||
|
|
||||||
content_type = request.headers.get("Content-Type")
|
content_type = request.headers.get("Content-Type")
|
||||||
validation_error = _validate_content_type(object_key, content_type)
|
validation_error = _validate_content_type(object_key, content_type)
|
||||||
if validation_error:
|
if validation_error:
|
||||||
return _error_response("InvalidArgument", validation_error, 400)
|
return _error_response("InvalidArgument", validation_error, 400)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
meta = storage.put_object(
|
meta = storage.put_object(
|
||||||
bucket_name,
|
bucket_name,
|
||||||
@@ -1922,10 +2283,21 @@ def object_handler(bucket_name: str, object_key: str):
|
|||||||
)
|
)
|
||||||
response = Response(status=200)
|
response = Response(status=200)
|
||||||
response.headers["ETag"] = f'"{meta.etag}"'
|
response.headers["ETag"] = f'"{meta.etag}"'
|
||||||
|
|
||||||
|
_notifications().emit_object_created(
|
||||||
|
bucket_name,
|
||||||
|
object_key,
|
||||||
|
size=meta.size,
|
||||||
|
etag=meta.etag,
|
||||||
|
request_id=getattr(g, "request_id", ""),
|
||||||
|
source_ip=request.remote_addr or "",
|
||||||
|
user_identity=principal.access_key if principal else "",
|
||||||
|
operation="Put",
|
||||||
|
)
|
||||||
|
|
||||||
if "S3ReplicationAgent" not in request.headers.get("User-Agent", ""):
|
if "S3ReplicationAgent" not in request.headers.get("User-Agent", ""):
|
||||||
_replication_manager().trigger_replication(bucket_name, object_key, action="write")
|
_replication_manager().trigger_replication(bucket_name, object_key, action="write")
|
||||||
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
if request.method in {"GET", "HEAD"}:
|
if request.method in {"GET", "HEAD"}:
|
||||||
@@ -2048,13 +2420,30 @@ def object_handler(bucket_name: str, object_key: str):
|
|||||||
_, error = _object_principal("delete", bucket_name, object_key)
|
_, error = _object_principal("delete", bucket_name, object_key)
|
||||||
if error:
|
if error:
|
||||||
return error
|
return error
|
||||||
|
|
||||||
|
bypass_governance = request.headers.get("x-amz-bypass-governance-retention", "").lower() == "true"
|
||||||
|
lock_service = _object_lock()
|
||||||
|
can_delete, lock_reason = lock_service.can_delete_object(bucket_name, object_key, bypass_governance=bypass_governance)
|
||||||
|
if not can_delete:
|
||||||
|
return _error_response("AccessDenied", lock_reason, 403)
|
||||||
|
|
||||||
storage.delete_object(bucket_name, object_key)
|
storage.delete_object(bucket_name, object_key)
|
||||||
|
lock_service.delete_object_lock_metadata(bucket_name, object_key)
|
||||||
current_app.logger.info("Object deleted", extra={"bucket": bucket_name, "key": object_key})
|
current_app.logger.info("Object deleted", extra={"bucket": bucket_name, "key": object_key})
|
||||||
|
|
||||||
|
principal, _ = _require_principal()
|
||||||
|
_notifications().emit_object_removed(
|
||||||
|
bucket_name,
|
||||||
|
object_key,
|
||||||
|
request_id=getattr(g, "request_id", ""),
|
||||||
|
source_ip=request.remote_addr or "",
|
||||||
|
user_identity=principal.access_key if principal else "",
|
||||||
|
)
|
||||||
|
|
||||||
user_agent = request.headers.get("User-Agent", "")
|
user_agent = request.headers.get("User-Agent", "")
|
||||||
if "S3ReplicationAgent" not in user_agent:
|
if "S3ReplicationAgent" not in user_agent:
|
||||||
_replication_manager().trigger_replication(bucket_name, object_key, action="delete")
|
_replication_manager().trigger_replication(bucket_name, object_key, action="delete")
|
||||||
|
|
||||||
return Response(status=204)
|
return Response(status=204)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
"""Ephemeral store for one-time secrets communicated to the UI."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import secrets
|
import secrets
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
"""Filesystem-backed object storage helpers."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
"""Authenticated HTML UI for browsing buckets and objects."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
"""Central location for the application version string."""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
APP_VERSION = "0.2.0"
|
APP_VERSION = "0.2.0"
|
||||||
|
|||||||
Reference in New Issue
Block a user