Release v0.1.3 #4

Merged
kqjy merged 9 commits from next into main 2025-12-03 04:14:57 +00:00
8 changed files with 719 additions and 22 deletions
Showing only changes of commit 9368715b16 - Show all commits

View File

@@ -266,5 +266,11 @@ class EncryptedObjectStorage:
def list_multipart_parts(self, bucket_name: str, upload_id: str): def list_multipart_parts(self, bucket_name: str, upload_id: str):
return self.storage.list_multipart_parts(bucket_name, upload_id) return self.storage.list_multipart_parts(bucket_name, upload_id)
def get_bucket_quota(self, bucket_name: str):
return self.storage.get_bucket_quota(bucket_name)
def set_bucket_quota(self, bucket_name: str, *, max_bytes=None, max_objects=None):
return self.storage.set_bucket_quota(bucket_name, max_bytes=max_bytes, max_objects=max_objects)
def _compute_etag(self, path: Path) -> str: def _compute_etag(self, path: Path) -> str:
return self.storage._compute_etag(path) return self.storage._compute_etag(path)

View File

@@ -129,6 +129,25 @@ class EntityTooLargeError(AppError):
status_code: int = 413 status_code: int = 413
@dataclass
class QuotaExceededAppError(AppError):
"""Bucket quota exceeded."""
code: str = "QuotaExceeded"
message: str = "The bucket quota has been exceeded"
status_code: int = 403
quota: Optional[Dict[str, Any]] = None
usage: Optional[Dict[str, int]] = None
def __post_init__(self):
if self.quota or self.usage:
self.details = {}
if self.quota:
self.details["quota"] = self.quota
if self.usage:
self.details["usage"] = self.usage
super().__post_init__()
def handle_app_error(error: AppError) -> Response: def handle_app_error(error: AppError) -> Response:
"""Handle application errors with appropriate response format.""" """Handle application errors with appropriate response format."""
log_extra = {"error_code": error.code} log_extra = {"error_code": error.code}
@@ -163,5 +182,6 @@ def register_error_handlers(app):
ObjectNotFoundError, InvalidObjectKeyError, ObjectNotFoundError, InvalidObjectKeyError,
AccessDeniedError, InvalidCredentialsError, AccessDeniedError, InvalidCredentialsError,
MalformedRequestError, InvalidArgumentError, EntityTooLargeError, MalformedRequestError, InvalidArgumentError, EntityTooLargeError,
QuotaExceededAppError,
]: ]:
app.register_error_handler(error_class, handle_app_error) app.register_error_handler(error_class, handle_app_error)

View File

@@ -18,7 +18,7 @@ from .bucket_policies import BucketPolicyStore
from .extensions import limiter from .extensions import limiter
from .iam import IamError, Principal from .iam import IamError, Principal
from .replication import ReplicationManager from .replication import ReplicationManager
from .storage import ObjectStorage, StorageError from .storage import ObjectStorage, StorageError, QuotaExceededError
s3_api_bp = Blueprint("s3_api", __name__) s3_api_bp = Blueprint("s3_api", __name__)
@@ -803,6 +803,7 @@ def _maybe_handle_bucket_subresource(bucket_name: str) -> Response | None:
"acl": _bucket_acl_handler, "acl": _bucket_acl_handler,
"versions": _bucket_list_versions_handler, "versions": _bucket_list_versions_handler,
"lifecycle": _bucket_lifecycle_handler, "lifecycle": _bucket_lifecycle_handler,
"quota": _bucket_quota_handler,
} }
requested = [key for key in handlers if key in request.args] requested = [key for key in handlers if key in request.args]
if not requested: if not requested:
@@ -1400,6 +1401,87 @@ def _parse_lifecycle_config(payload: bytes) -> list:
return rules return rules
def _bucket_quota_handler(bucket_name: str) -> Response:
"""Handle bucket quota configuration (GET/PUT/DELETE /<bucket>?quota)."""
if request.method not in {"GET", "PUT", "DELETE"}:
return _method_not_allowed(["GET", "PUT", "DELETE"])
principal, error = _require_principal()
if error:
return error
try:
_authorize_action(principal, bucket_name, "policy")
except IamError as exc:
return _error_response("AccessDenied", str(exc), 403)
storage = _storage()
if not storage.bucket_exists(bucket_name):
return _error_response("NoSuchBucket", "Bucket does not exist", 404)
if request.method == "GET":
quota = storage.get_bucket_quota(bucket_name)
if not quota:
return _error_response("NoSuchQuotaConfiguration", "No quota configuration found", 404)
# Return as JSON for simplicity (not a standard S3 API)
stats = storage.bucket_stats(bucket_name)
return jsonify({
"quota": quota,
"usage": {
"bytes": stats.get("bytes", 0),
"objects": stats.get("objects", 0),
}
})
if request.method == "DELETE":
try:
storage.set_bucket_quota(bucket_name, max_size_bytes=None, max_objects=None)
except StorageError as exc:
return _error_response("NoSuchBucket", str(exc), 404)
current_app.logger.info("Bucket quota deleted", extra={"bucket": bucket_name})
return Response(status=204)
# PUT
payload = request.get_json(silent=True)
if not payload:
return _error_response("MalformedRequest", "Request body must be JSON with quota limits", 400)
max_size_bytes = payload.get("max_size_bytes")
max_objects = payload.get("max_objects")
if max_size_bytes is None and max_objects is None:
return _error_response("InvalidArgument", "At least one of max_size_bytes or max_objects is required", 400)
# Validate types
if max_size_bytes is not None:
try:
max_size_bytes = int(max_size_bytes)
if max_size_bytes < 0:
raise ValueError("must be non-negative")
except (TypeError, ValueError) as exc:
return _error_response("InvalidArgument", f"max_size_bytes {exc}", 400)
if max_objects is not None:
try:
max_objects = int(max_objects)
if max_objects < 0:
raise ValueError("must be non-negative")
except (TypeError, ValueError) as exc:
return _error_response("InvalidArgument", f"max_objects {exc}", 400)
try:
storage.set_bucket_quota(bucket_name, max_size_bytes=max_size_bytes, max_objects=max_objects)
except StorageError as exc:
return _error_response("NoSuchBucket", str(exc), 404)
current_app.logger.info(
"Bucket quota updated",
extra={"bucket": bucket_name, "max_size_bytes": max_size_bytes, "max_objects": max_objects}
)
return Response(status=204)
def _bulk_delete_handler(bucket_name: str) -> Response: def _bulk_delete_handler(bucket_name: str) -> Response:
principal, error = _require_principal() principal, error = _require_principal()
if error: if error:
@@ -1749,6 +1831,8 @@ def object_handler(bucket_name: str, object_key: str):
stream, stream,
metadata=metadata or None, metadata=metadata or None,
) )
except QuotaExceededError as exc:
return _error_response("QuotaExceeded", str(exc), 403)
except StorageError as exc: except StorageError as exc:
message = str(exc) message = str(exc)
if "Bucket" in message: if "Bucket" in message:
@@ -2256,6 +2340,8 @@ def _complete_multipart_upload(bucket_name: str, object_key: str) -> Response:
try: try:
meta = _storage().complete_multipart_upload(bucket_name, upload_id, parts) meta = _storage().complete_multipart_upload(bucket_name, upload_id, parts)
except QuotaExceededError as exc:
return _error_response("QuotaExceeded", str(exc), 403)
except StorageError as exc: except StorageError as exc:
if "NoSuchBucket" in str(exc): if "NoSuchBucket" in str(exc):
return _error_response("NoSuchBucket", str(exc), 404) return _error_response("NoSuchBucket", str(exc), 404)

View File

@@ -75,6 +75,15 @@ class StorageError(RuntimeError):
"""Raised when the storage layer encounters an unrecoverable problem.""" """Raised when the storage layer encounters an unrecoverable problem."""
class QuotaExceededError(StorageError):
"""Raised when an operation would exceed bucket quota limits."""
def __init__(self, message: str, quota: Dict[str, Any], usage: Dict[str, int]):
super().__init__(message)
self.quota = quota
self.usage = usage
@dataclass @dataclass
class ObjectMeta: class ObjectMeta:
key: str key: str
@@ -169,16 +178,38 @@ class ObjectStorage:
object_count = 0 object_count = 0
total_bytes = 0 total_bytes = 0
version_count = 0
version_bytes = 0
# Count current objects in the bucket folder
for path in bucket_path.rglob("*"): for path in bucket_path.rglob("*"):
if path.is_file(): if path.is_file():
rel = path.relative_to(bucket_path) rel = path.relative_to(bucket_path)
if rel.parts and rel.parts[0] in self.INTERNAL_FOLDERS: if not rel.parts:
continue continue
stat = path.stat() top_folder = rel.parts[0]
object_count += 1 if top_folder not in self.INTERNAL_FOLDERS:
total_bytes += stat.st_size stat = path.stat()
object_count += 1
total_bytes += stat.st_size
stats = {"objects": object_count, "bytes": total_bytes} # Count archived versions in the system folder
versions_root = self._bucket_versions_root(bucket_name)
if versions_root.exists():
for path in versions_root.rglob("*.bin"):
if path.is_file():
stat = path.stat()
version_count += 1
version_bytes += stat.st_size
stats = {
"objects": object_count,
"bytes": total_bytes,
"version_count": version_count,
"version_bytes": version_bytes,
"total_objects": object_count + version_count, # All objects including versions
"total_bytes": total_bytes + version_bytes, # All storage including versions
}
try: try:
cache_path.parent.mkdir(parents=True, exist_ok=True) cache_path.parent.mkdir(parents=True, exist_ok=True)
@@ -243,6 +274,7 @@ class ObjectStorage:
stream: BinaryIO, stream: BinaryIO,
*, *,
metadata: Optional[Dict[str, str]] = None, metadata: Optional[Dict[str, str]] = None,
enforce_quota: bool = True,
) -> ObjectMeta: ) -> ObjectMeta:
bucket_path = self._bucket_path(bucket_name) bucket_path = self._bucket_path(bucket_name)
if not bucket_path.exists(): if not bucket_path.exists():
@@ -253,12 +285,52 @@ class ObjectStorage:
destination = bucket_path / safe_key destination = bucket_path / safe_key
destination.parent.mkdir(parents=True, exist_ok=True) destination.parent.mkdir(parents=True, exist_ok=True)
if self._is_versioning_enabled(bucket_path) and destination.exists(): # Check if this is an overwrite (won't add to object count)
is_overwrite = destination.exists()
existing_size = destination.stat().st_size if is_overwrite else 0
if self._is_versioning_enabled(bucket_path) and is_overwrite:
self._archive_current_version(bucket_id, safe_key, reason="overwrite") self._archive_current_version(bucket_id, safe_key, reason="overwrite")
checksum = hashlib.md5() # Write to temp file first to get actual size
with destination.open("wb") as target: tmp_dir = self._system_root_path() / self.SYSTEM_TMP_DIR
shutil.copyfileobj(_HashingReader(stream, checksum), target) tmp_dir.mkdir(parents=True, exist_ok=True)
tmp_path = tmp_dir / f"{uuid.uuid4().hex}.tmp"
try:
checksum = hashlib.md5()
with tmp_path.open("wb") as target:
shutil.copyfileobj(_HashingReader(stream, checksum), target)
new_size = tmp_path.stat().st_size
# Check quota before finalizing
if enforce_quota:
# Calculate net change (new size minus size being replaced)
size_delta = new_size - existing_size
object_delta = 0 if is_overwrite else 1
quota_check = self.check_quota(
bucket_name,
additional_bytes=max(0, size_delta),
additional_objects=object_delta,
)
if not quota_check["allowed"]:
raise QuotaExceededError(
quota_check["message"] or "Quota exceeded",
quota_check["quota"],
quota_check["usage"],
)
# Move to final destination
shutil.move(str(tmp_path), str(destination))
finally:
# Clean up temp file if it still exists
try:
tmp_path.unlink(missing_ok=True)
except OSError:
pass
stat = destination.stat() stat = destination.stat()
if metadata: if metadata:
@@ -424,6 +496,124 @@ class ObjectStorage:
bucket_path = self._require_bucket_path(bucket_name) bucket_path = self._require_bucket_path(bucket_name)
self._set_bucket_config_entry(bucket_path.name, "lifecycle", rules) self._set_bucket_config_entry(bucket_path.name, "lifecycle", rules)
def get_bucket_quota(self, bucket_name: str) -> Dict[str, Any]:
"""Get quota configuration for bucket.
Returns:
Dict with 'max_bytes' and 'max_objects' (None if unlimited).
"""
bucket_path = self._require_bucket_path(bucket_name)
config = self._read_bucket_config(bucket_path.name)
quota = config.get("quota")
if isinstance(quota, dict):
return {
"max_bytes": quota.get("max_bytes"),
"max_objects": quota.get("max_objects"),
}
return {"max_bytes": None, "max_objects": None}
def set_bucket_quota(
self,
bucket_name: str,
*,
max_bytes: Optional[int] = None,
max_objects: Optional[int] = None,
) -> None:
"""Set quota limits for a bucket.
Args:
bucket_name: Name of the bucket
max_bytes: Maximum total size in bytes (None to remove limit)
max_objects: Maximum number of objects (None to remove limit)
"""
bucket_path = self._require_bucket_path(bucket_name)
if max_bytes is None and max_objects is None:
# Remove quota entirely
self._set_bucket_config_entry(bucket_path.name, "quota", None)
return
quota: Dict[str, Any] = {}
if max_bytes is not None:
if max_bytes < 0:
raise StorageError("max_bytes must be non-negative")
quota["max_bytes"] = max_bytes
if max_objects is not None:
if max_objects < 0:
raise StorageError("max_objects must be non-negative")
quota["max_objects"] = max_objects
self._set_bucket_config_entry(bucket_path.name, "quota", quota)
def check_quota(
self,
bucket_name: str,
additional_bytes: int = 0,
additional_objects: int = 0,
) -> Dict[str, Any]:
"""Check if an operation would exceed bucket quota.
Args:
bucket_name: Name of the bucket
additional_bytes: Bytes that would be added
additional_objects: Objects that would be added
Returns:
Dict with 'allowed' (bool), 'quota' (current limits),
'usage' (current usage), and 'message' (if not allowed).
"""
quota = self.get_bucket_quota(bucket_name)
if not quota:
return {
"allowed": True,
"quota": None,
"usage": None,
"message": None,
}
# Get current stats (uses cache when available)
stats = self.bucket_stats(bucket_name)
# Use totals which include versions for quota enforcement
current_bytes = stats.get("total_bytes", stats.get("bytes", 0))
current_objects = stats.get("total_objects", stats.get("objects", 0))
result = {
"allowed": True,
"quota": quota,
"usage": {
"bytes": current_bytes,
"objects": current_objects,
"version_count": stats.get("version_count", 0),
"version_bytes": stats.get("version_bytes", 0),
},
"message": None,
}
max_bytes_limit = quota.get("max_bytes")
max_objects = quota.get("max_objects")
if max_bytes_limit is not None:
projected_bytes = current_bytes + additional_bytes
if projected_bytes > max_bytes_limit:
result["allowed"] = False
result["message"] = (
f"Quota exceeded: adding {additional_bytes} bytes would result in "
f"{projected_bytes} bytes, exceeding limit of {max_bytes_limit} bytes"
)
return result
if max_objects is not None:
projected_objects = current_objects + additional_objects
if projected_objects > max_objects:
result["allowed"] = False
result["message"] = (
f"Quota exceeded: adding {additional_objects} objects would result in "
f"{projected_objects} objects, exceeding limit of {max_objects} objects"
)
return result
return result
def get_object_tags(self, bucket_name: str, object_key: str) -> List[Dict[str, str]]: def get_object_tags(self, bucket_name: str, object_key: str) -> List[Dict[str, str]]:
"""Get tags for an object.""" """Get tags for an object."""
bucket_path = self._bucket_path(bucket_name) bucket_path = self._bucket_path(bucket_name)
@@ -540,6 +730,7 @@ class ObjectStorage:
else: else:
self._delete_metadata(bucket_id, safe_key) self._delete_metadata(bucket_id, safe_key)
stat = destination.stat() stat = destination.stat()
self._invalidate_bucket_stats_cache(bucket_id)
return ObjectMeta( return ObjectMeta(
key=safe_key.as_posix(), key=safe_key.as_posix(),
size=stat.st_size, size=stat.st_size,
@@ -688,6 +879,7 @@ class ObjectStorage:
bucket_name: str, bucket_name: str,
upload_id: str, upload_id: str,
ordered_parts: List[Dict[str, Any]], ordered_parts: List[Dict[str, Any]],
enforce_quota: bool = True,
) -> ObjectMeta: ) -> ObjectMeta:
if not ordered_parts: if not ordered_parts:
raise StorageError("parts list required") raise StorageError("parts list required")
@@ -698,6 +890,7 @@ class ObjectStorage:
if not parts_map: if not parts_map:
raise StorageError("No uploaded parts found") raise StorageError("No uploaded parts found")
validated: List[tuple[int, Dict[str, Any]]] = [] validated: List[tuple[int, Dict[str, Any]]] = []
total_size = 0
for part in ordered_parts: for part in ordered_parts:
raw_number = part.get("part_number") raw_number = part.get("part_number")
if raw_number is None: if raw_number is None:
@@ -717,10 +910,33 @@ class ObjectStorage:
if supplied_etag and record.get("etag") and supplied_etag.strip('"') != record["etag"]: if supplied_etag and record.get("etag") and supplied_etag.strip('"') != record["etag"]:
raise StorageError(f"ETag mismatch for part {number}") raise StorageError(f"ETag mismatch for part {number}")
validated.append((number, record)) validated.append((number, record))
total_size += record.get("size", 0)
validated.sort(key=lambda entry: entry[0]) validated.sort(key=lambda entry: entry[0])
safe_key = self._sanitize_object_key(manifest["object_key"]) safe_key = self._sanitize_object_key(manifest["object_key"])
destination = bucket_path / safe_key destination = bucket_path / safe_key
# Check if this is an overwrite
is_overwrite = destination.exists()
existing_size = destination.stat().st_size if is_overwrite else 0
# Check quota before writing
if enforce_quota:
size_delta = total_size - existing_size
object_delta = 0 if is_overwrite else 1
quota_check = self.check_quota(
bucket_name,
additional_bytes=max(0, size_delta),
additional_objects=object_delta,
)
if not quota_check["allowed"]:
raise QuotaExceededError(
quota_check["message"] or "Quota exceeded",
quota_check["quota"],
quota_check["usage"],
)
destination.parent.mkdir(parents=True, exist_ok=True) destination.parent.mkdir(parents=True, exist_ok=True)
lock_file_path = self._system_bucket_root(bucket_id) / "locks" / f"{safe_key.as_posix().replace('/', '_')}.lock" lock_file_path = self._system_bucket_root(bucket_id) / "locks" / f"{safe_key.as_posix().replace('/', '_')}.lock"

View File

@@ -260,9 +260,9 @@ def buckets_overview():
visible_buckets.append({ visible_buckets.append({
"meta": bucket, "meta": bucket,
"summary": { "summary": {
"objects": stats["objects"], "objects": stats["total_objects"],
"total_bytes": stats["bytes"], "total_bytes": stats["total_bytes"],
"human_size": _format_bytes(stats["bytes"]), "human_size": _format_bytes(stats["total_bytes"]),
}, },
"access_label": access_label, "access_label": access_label,
"access_badge": access_badge, "access_badge": access_badge,
@@ -372,6 +372,16 @@ def bucket_detail(bucket_name: str):
encryption_enabled = current_app.config.get("ENCRYPTION_ENABLED", False) encryption_enabled = current_app.config.get("ENCRYPTION_ENABLED", False)
can_manage_encryption = can_manage_versioning # Same as other bucket properties can_manage_encryption = can_manage_versioning # Same as other bucket properties
# Quota settings (admin only)
bucket_quota = storage.get_bucket_quota(bucket_name)
bucket_stats = storage.bucket_stats(bucket_name)
can_manage_quota = False
try:
_iam().authorize(principal, None, "iam:list_users")
can_manage_quota = True
except IamError:
pass
return render_template( return render_template(
"bucket_detail.html", "bucket_detail.html",
bucket_name=bucket_name, bucket_name=bucket_name,
@@ -392,6 +402,9 @@ def bucket_detail(bucket_name: str):
kms_keys=kms_keys, kms_keys=kms_keys,
kms_enabled=kms_enabled, kms_enabled=kms_enabled,
encryption_enabled=encryption_enabled, encryption_enabled=encryption_enabled,
bucket_quota=bucket_quota,
bucket_stats=bucket_stats,
can_manage_quota=can_manage_quota,
) )
@@ -925,6 +938,71 @@ def update_bucket_versioning(bucket_name: str):
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties")) return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
@ui_bp.post("/buckets/<bucket_name>/quota")
def update_bucket_quota(bucket_name: str):
"""Update bucket quota configuration (admin only)."""
principal = _current_principal()
# Quota management is admin-only
is_admin = False
try:
_iam().authorize(principal, None, "iam:list_users")
is_admin = True
except IamError:
pass
if not is_admin:
flash("Only administrators can manage bucket quotas", "danger")
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
action = request.form.get("action", "set")
if action == "remove":
try:
_storage().set_bucket_quota(bucket_name, max_bytes=None, max_objects=None)
flash("Bucket quota removed", "info")
except StorageError as exc:
flash(_friendly_error_message(exc), "danger")
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
# Parse quota values
max_mb_str = request.form.get("max_mb", "").strip()
max_objects_str = request.form.get("max_objects", "").strip()
max_bytes = None
max_objects = None
if max_mb_str:
try:
max_mb = int(max_mb_str)
if max_mb < 1:
raise ValueError("Size must be at least 1 MB")
max_bytes = max_mb * 1024 * 1024 # Convert MB to bytes
except ValueError as exc:
flash(f"Invalid size value: {exc}", "danger")
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
if max_objects_str:
try:
max_objects = int(max_objects_str)
if max_objects < 0:
raise ValueError("Object count must be non-negative")
except ValueError as exc:
flash(f"Invalid object count: {exc}", "danger")
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
try:
_storage().set_bucket_quota(bucket_name, max_bytes=max_bytes, max_objects=max_objects)
if max_bytes is None and max_objects is None:
flash("Bucket quota removed", "info")
else:
flash("Bucket quota updated", "success")
except StorageError as exc:
flash(_friendly_error_message(exc), "danger")
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
@ui_bp.post("/buckets/<bucket_name>/encryption") @ui_bp.post("/buckets/<bucket_name>/encryption")
def update_bucket_encryption(bucket_name: str): def update_bucket_encryption(bucket_name: str):
"""Update bucket default encryption configuration.""" """Update bucket default encryption configuration."""
@@ -1438,13 +1516,16 @@ def metrics_dashboard():
total_objects = 0 total_objects = 0
total_bytes_used = 0 total_bytes_used = 0
total_versions = 0
# Note: Uses cached stats from storage layer to improve performance # Note: Uses cached stats from storage layer to improve performance
cache_ttl = current_app.config.get("BUCKET_STATS_CACHE_TTL", 60) cache_ttl = current_app.config.get("BUCKET_STATS_CACHE_TTL", 60)
for bucket in buckets: for bucket in buckets:
stats = storage.bucket_stats(bucket.name, cache_ttl=cache_ttl) stats = storage.bucket_stats(bucket.name, cache_ttl=cache_ttl)
total_objects += stats["objects"] # Use totals which include archived versions
total_bytes_used += stats["bytes"] total_objects += stats.get("total_objects", stats.get("objects", 0))
total_bytes_used += stats.get("total_bytes", stats.get("bytes", 0))
total_versions += stats.get("version_count", 0)
return render_template( return render_template(
"metrics.html", "metrics.html",
@@ -1465,6 +1546,7 @@ def metrics_dashboard():
app={ app={
"buckets": total_buckets, "buckets": total_buckets,
"objects": total_objects, "objects": total_objects,
"versions": total_versions,
"storage_used": _format_bytes(total_bytes_used), "storage_used": _format_bytes(total_bytes_used),
"storage_raw": total_bytes_used, "storage_raw": total_bytes_used,
} }

76
docs.md
View File

@@ -340,7 +340,71 @@ To verify an object is encrypted:
2. Look for `.meta` files containing encryption metadata 2. Look for `.meta` files containing encryption metadata
3. Download via the API/UI - the object should be automatically decrypted 3. Download via the API/UI - the object should be automatically decrypted
## 8. Site Replication ## 8. Bucket Quotas
MyFSIO supports **storage quotas** to limit how much data a bucket can hold. Quotas are enforced on uploads and multipart completions.
### Quota Types
| Limit | Description |
|-------|-------------|
| **Max Size (MB)** | Maximum total storage in megabytes (includes current objects + archived versions) |
| **Max Objects** | Maximum number of objects (includes current objects + archived versions) |
### Managing Quotas (Admin Only)
Quota management is restricted to administrators (users with `iam:*` or `iam:list_users` permissions).
#### Via UI
1. Navigate to your bucket in the UI
2. Click the **Properties** tab
3. Find the **Storage Quota** card
4. Enter limits:
- **Max Size (MB)**: Leave empty for unlimited
- **Max Objects**: Leave empty for unlimited
5. Click **Update Quota**
To remove a quota, click **Remove Quota**.
#### Via API
```bash
# Set quota (max 100MB, max 1000 objects)
curl -X PUT "http://localhost:5000/bucket/<bucket>?quota" \
-H "Content-Type: application/json" \
-H "X-Access-Key: ..." -H "X-Secret-Key: ..." \
-d '{"max_bytes": 104857600, "max_objects": 1000}'
# Get current quota
curl "http://localhost:5000/bucket/<bucket>?quota" \
-H "X-Access-Key: ..." -H "X-Secret-Key: ..."
# Remove quota
curl -X PUT "http://localhost:5000/bucket/<bucket>?quota" \
-H "Content-Type: application/json" \
-H "X-Access-Key: ..." -H "X-Secret-Key: ..." \
-d '{"max_bytes": null, "max_objects": null}'
```
### Quota Behavior
- **Version Counting**: When versioning is enabled, archived versions count toward the quota (similar to MinIO behavior)
- **Enforcement Points**: Quotas are checked during `PUT` object and `CompleteMultipartUpload` operations
- **Error Response**: When quota is exceeded, the API returns `HTTP 400` with error code `QuotaExceeded`
- **Visibility**: All users can view quota usage in the bucket detail page, but only admins can modify quotas
### Example Error
```xml
<Error>
<Code>QuotaExceeded</Code>
<Message>Bucket quota exceeded: storage limit reached</Message>
<BucketName>my-bucket</BucketName>
</Error>
```
## 9. Site Replication
### Permission Model ### Permission Model
@@ -477,7 +541,7 @@ To set up two-way replication (Server A ↔ Server B):
**Note**: Deleting a bucket will automatically remove its associated replication configuration. **Note**: Deleting a bucket will automatically remove its associated replication configuration.
## 9. Running Tests ## 11. Running Tests
```bash ```bash
pytest -q pytest -q
@@ -487,7 +551,7 @@ The suite now includes a boto3 integration test that spins up a live HTTP server
The suite covers bucket CRUD, presigned downloads, bucket policy enforcement, and regression tests for anonymous reads when a Public policy is attached. The suite covers bucket CRUD, presigned downloads, bucket policy enforcement, and regression tests for anonymous reads when a Public policy is attached.
## 10. Troubleshooting ## 12. Troubleshooting
| Symptom | Likely Cause | Fix | | Symptom | Likely Cause | Fix |
| --- | --- | --- | | --- | --- | --- |
@@ -496,7 +560,7 @@ The suite covers bucket CRUD, presigned downloads, bucket policy enforcement, an
| Presign modal errors with 403 | IAM user lacks `read/write/delete` for target bucket or bucket policy denies | Update IAM inline policies or remove conflicting deny statements. | | Presign modal errors with 403 | IAM user lacks `read/write/delete` for target bucket or bucket policy denies | Update IAM inline policies or remove conflicting deny statements. |
| Large upload rejected immediately | File exceeds `MAX_UPLOAD_SIZE` | Increase env var or shrink object. | | Large upload rejected immediately | File exceeds `MAX_UPLOAD_SIZE` | Increase env var or shrink object. |
## 11. API Matrix ## 13. API Matrix
``` ```
GET / # List buckets GET / # List buckets
@@ -510,9 +574,11 @@ POST /presign/<bucket>/<key> # Generate SigV4 URL
GET /bucket-policy/<bucket> # Fetch policy GET /bucket-policy/<bucket> # Fetch policy
PUT /bucket-policy/<bucket> # Upsert policy PUT /bucket-policy/<bucket> # Upsert policy
DELETE /bucket-policy/<bucket> # Delete policy DELETE /bucket-policy/<bucket> # Delete policy
GET /<bucket>?quota # Get bucket quota
PUT /<bucket>?quota # Set bucket quota (admin only)
``` ```
## 12. Next Steps ## 14. Next Steps
- Tailor IAM + policy JSON files for team-ready presets. - Tailor IAM + policy JSON files for team-ready presets.
- Wrap `run_api.py` with gunicorn or another WSGI server for long-running workloads. - Wrap `run_api.py` with gunicorn or another WSGI server for long-running workloads.

View File

@@ -730,6 +730,158 @@
</div> </div>
</div> </div>
{% endif %} {% endif %}
<!-- Storage Quota Card -->
<div class="card shadow-sm mt-4" id="bucket-quota-card">
<div class="card-header d-flex align-items-center">
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" fill="currentColor" class="text-primary me-2" viewBox="0 0 16 16">
<path d="M1 2.5A1.5 1.5 0 0 1 2.5 1h3A1.5 1.5 0 0 1 7 2.5v3A1.5 1.5 0 0 1 5.5 7h-3A1.5 1.5 0 0 1 1 5.5v-3zM2.5 2a.5.5 0 0 0-.5.5v3a.5.5 0 0 0 .5.5h3a.5.5 0 0 0 .5-.5v-3a.5.5 0 0 0-.5-.5h-3zm6.5.5A1.5 1.5 0 0 1 10.5 1h3A1.5 1.5 0 0 1 15 2.5v3A1.5 1.5 0 0 1 13.5 7h-3A1.5 1.5 0 0 1 9 5.5v-3zm1.5-.5a.5.5 0 0 0-.5.5v3a.5.5 0 0 0 .5.5h3a.5.5 0 0 0 .5-.5v-3a.5.5 0 0 0-.5-.5h-3zM1 10.5A1.5 1.5 0 0 1 2.5 9h3A1.5 1.5 0 0 1 7 10.5v3A1.5 1.5 0 0 1 5.5 15h-3A1.5 1.5 0 0 1 1 13.5v-3zm1.5-.5a.5.5 0 0 0-.5.5v3a.5.5 0 0 0 .5.5h3a.5.5 0 0 0 .5-.5v-3a.5.5 0 0 0-.5-.5h-3zm6.5.5A1.5 1.5 0 0 1 10.5 9h3a1.5 1.5 0 0 1 1.5 1.5v3a1.5 1.5 0 0 1-1.5 1.5h-3A1.5 1.5 0 0 1 9 13.5v-3zm1.5-.5a.5.5 0 0 0-.5.5v3a.5.5 0 0 0 .5.5h3a.5.5 0 0 0 .5-.5v-3a.5.5 0 0 0-.5-.5h-3z"/>
</svg>
<span class="fw-semibold">Storage Quota</span>
</div>
<div class="card-body">
{% set max_bytes = bucket_quota.get('max_bytes') %}
{% set max_objects = bucket_quota.get('max_objects') %}
{% set has_quota = max_bytes is not none or max_objects is not none %}
{% set current_objects = bucket_stats.get('objects', 0) %}
{% set version_count = bucket_stats.get('version_count', 0) %}
{% set total_objects = bucket_stats.get('total_objects', current_objects) %}
{% set current_bytes = bucket_stats.get('bytes', 0) %}
{% set version_bytes = bucket_stats.get('version_bytes', 0) %}
{% set total_bytes = bucket_stats.get('total_bytes', current_bytes) %}
<!-- Current Usage Display -->
<div class="mb-4">
<h6 class="small fw-semibold mb-3">Current Usage</h6>
<div class="row g-3">
<div class="col-6">
<div class="border rounded p-3 text-center">
<div class="fs-4 fw-bold text-primary">{{ total_objects }}</div>
<div class="small text-muted">Total Objects</div>
{% if max_objects is not none %}
<div class="progress mt-2" style="height: 4px;">
{% set obj_pct = (total_objects / max_objects * 100) | int if max_objects > 0 else 0 %}
<div class="progress-bar {% if obj_pct >= 90 %}bg-danger{% elif obj_pct >= 75 %}bg-warning{% else %}bg-success{% endif %}" style="width: {{ [obj_pct, 100] | min }}%"></div>
</div>
<div class="small text-muted mt-1">{{ obj_pct }}% of {{ max_objects }} limit</div>
{% else %}
<div class="small text-muted mt-2">No limit</div>
{% endif %}
{% if version_count > 0 %}
<div class="small text-muted mt-1">
<span class="text-body-secondary">({{ current_objects }} current + {{ version_count }} versions)</span>
</div>
{% endif %}
</div>
</div>
<div class="col-6">
<div class="border rounded p-3 text-center">
<div class="fs-4 fw-bold text-primary">{{ total_bytes | filesizeformat }}</div>
<div class="small text-muted">Total Storage</div>
{% if max_bytes is not none %}
<div class="progress mt-2" style="height: 4px;">
{% set bytes_pct = (total_bytes / max_bytes * 100) | int if max_bytes > 0 else 0 %}
<div class="progress-bar {% if bytes_pct >= 90 %}bg-danger{% elif bytes_pct >= 75 %}bg-warning{% else %}bg-success{% endif %}" style="width: {{ [bytes_pct, 100] | min }}%"></div>
</div>
<div class="small text-muted mt-1">{{ bytes_pct }}% of {{ max_bytes | filesizeformat }} limit</div>
{% else %}
<div class="small text-muted mt-2">No limit</div>
{% endif %}
{% if version_bytes > 0 %}
<div class="small text-muted mt-1">
<span class="text-body-secondary">({{ current_bytes | filesizeformat }} current + {{ version_bytes | filesizeformat }} versions)</span>
</div>
{% endif %}
</div>
</div>
</div>
</div>
{% if has_quota %}
<!-- Quota Enabled State -->
<div class="alert alert-info d-flex align-items-start mb-4" role="alert">
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="me-2 flex-shrink-0" viewBox="0 0 16 16">
<path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm.93-9.412-1 4.705c-.07.34.029.533.304.533.194 0 .487-.07.686-.246l-.088.416c-.287.346-.92.598-1.465.598-.703 0-1.002-.422-.808-1.319l.738-3.468c.064-.293.006-.399-.287-.47l-.451-.081.082-.381 2.29-.287zM8 5.5a1 1 0 1 1 0-2 1 1 0 0 1 0 2z"/>
</svg>
<div>
<strong>Storage quota enabled</strong>
<p class="mb-0 small">
{% if max_bytes is not none and max_objects is not none %}
Limited to {{ max_bytes | filesizeformat }} and {{ max_objects }} objects.
{% elif max_bytes is not none %}
Limited to {{ max_bytes | filesizeformat }} storage.
{% else %}
Limited to {{ max_objects }} objects.
{% endif %}
</p>
</div>
</div>
{% else %}
<!-- Quota Disabled State -->
<div class="alert alert-secondary d-flex align-items-start mb-4" role="alert">
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="me-2 flex-shrink-0" viewBox="0 0 16 16">
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
<path d="M4.646 4.646a.5.5 0 0 1 .708 0L8 7.293l2.646-2.647a.5.5 0 0 1 .708.708L8.707 8l2.647 2.646a.5.5 0 0 1-.708.708L8 8.707l-2.646 2.647a.5.5 0 0 1-.708-.708L7.293 8 4.646 5.354a.5.5 0 0 1 0-.708z"/>
</svg>
<div>
<strong>No storage quota</strong>
<p class="mb-0 small">This bucket has no storage or object count limits. Set limits below to control usage.</p>
</div>
</div>
{% endif %}
{% if can_manage_quota %}
<form method="post" action="{{ url_for('ui.update_bucket_quota', bucket_name=bucket_name) }}" id="quotaForm">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<!-- Max Storage -->
<div class="mb-3">
<label for="max_mb" class="form-label fw-medium">Maximum Storage Size</label>
<div class="input-group">
<input type="number" class="form-control" id="max_mb" name="max_mb"
value="{{ (max_bytes / 1048576) | int if max_bytes is not none else '' }}"
min="1" step="1" placeholder="Unlimited">
<span class="input-group-text">MB</span>
</div>
<div class="form-text">Minimum 1 MB. Leave empty for unlimited.</div>
</div>
<!-- Max Objects -->
<div class="mb-4">
<label for="max_objects" class="form-label fw-medium">Maximum Object Count</label>
<input type="number" class="form-control" id="max_objects" name="max_objects"
value="{{ max_objects if max_objects is not none else '' }}"
min="0" step="1" placeholder="Unlimited">
<div class="form-text">Maximum number of objects allowed. Leave empty for unlimited.</div>
</div>
<div class="d-flex gap-2 flex-wrap">
<button class="btn btn-primary" type="submit">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
<path d="M12.736 3.97a.733.733 0 0 1 1.047 0c.286.289.29.756.01 1.05L7.88 12.01a.733.733 0 0 1-1.065.02L3.217 8.384a.757.757 0 0 1 0-1.06.733.733 0 0 1 1.047 0l3.052 3.093 5.4-6.425a.247.247 0 0 1 .02-.022Z"/>
</svg>
Save Quota Settings
</button>
{% if has_quota %}
<button type="submit" class="btn btn-outline-danger" id="removeQuotaBtn" name="action" value="remove">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
<path d="M4.646 4.646a.5.5 0 0 1 .708 0L8 7.293l2.646-2.647a.5.5 0 0 1 .708.708L8.707 8l2.647 2.646a.5.5 0 0 1-.708.708L8 8.707l-2.646 2.647a.5.5 0 0 1-.708-.708L7.293 8 4.646 5.354a.5.5 0 0 1 0-.708z"/>
</svg>
Remove Quota
</button>
{% endif %}
</div>
</form>
{% else %}
<div class="text-center py-3">
<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" fill="currentColor" class="text-muted mb-2" viewBox="0 0 16 16">
<path d="M8 1a2 2 0 0 1 2 2v4H6V3a2 2 0 0 1 2-2zm3 6V3a3 3 0 0 0-6 0v4a2 2 0 0 0-2 2v5a2 2 0 0 0 2 2h6a2 2 0 0 0 2-2V9a2 2 0 0 0-2-2z"/>
</svg>
<p class="text-muted mb-0 small">You do not have permission to modify quota settings for this bucket.</p>
</div>
{% endif %}
</div>
</div>
</div> </div>
<!-- Sidebar --> <!-- Sidebar -->

View File

@@ -418,10 +418,78 @@ s3.complete_multipart_upload(
</p> </p>
</div> </div>
</article> </article>
<article id="quotas" class="card shadow-sm docs-section">
<div class="card-body">
<div class="d-flex align-items-center gap-2 mb-3">
<span class="docs-section-kicker">10</span>
<h2 class="h4 mb-0">Bucket Quotas</h2>
</div>
<p class="text-muted">Limit how much data a bucket can hold using storage quotas. Quotas are enforced on uploads and multipart completions.</p>
<h3 class="h6 text-uppercase text-muted mt-4">Quota Types</h3>
<div class="table-responsive mb-3">
<table class="table table-sm table-bordered small">
<thead class="table-light">
<tr>
<th>Limit</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td><strong>Max Size (MB)</strong></td>
<td>Maximum total storage in megabytes (includes current objects + archived versions)</td>
</tr>
<tr>
<td><strong>Max Objects</strong></td>
<td>Maximum number of objects (includes current objects + archived versions)</td>
</tr>
</tbody>
</table>
</div>
<h3 class="h6 text-uppercase text-muted mt-4">Managing Quotas (Admin Only)</h3>
<p class="small text-muted">Quota management is restricted to administrators (users with <code>iam:*</code> permissions).</p>
<ol class="docs-steps mb-3">
<li>Navigate to your bucket → <strong>Properties</strong> tab → <strong>Storage Quota</strong> card.</li>
<li>Enter limits: <strong>Max Size (MB)</strong> and/or <strong>Max Objects</strong>. Leave empty for unlimited.</li>
<li>Click <strong>Update Quota</strong> to save, or <strong>Remove Quota</strong> to clear limits.</li>
</ol>
<h3 class="h6 text-uppercase text-muted mt-4">API Usage</h3>
<pre class="mb-3"><code class="language-bash"># Set quota (max 100MB, max 1000 objects)
curl -X PUT "{{ api_base }}/bucket/&lt;bucket&gt;?quota" \
-H "Content-Type: application/json" \
-H "X-Access-Key: &lt;key&gt;" -H "X-Secret-Key: &lt;secret&gt;" \
-d '{"max_bytes": 104857600, "max_objects": 1000}'
# Get current quota
curl "{{ api_base }}/bucket/&lt;bucket&gt;?quota" \
-H "X-Access-Key: &lt;key&gt;" -H "X-Secret-Key: &lt;secret&gt;"
# Remove quota
curl -X PUT "{{ api_base }}/bucket/&lt;bucket&gt;?quota" \
-H "Content-Type: application/json" \
-H "X-Access-Key: &lt;key&gt;" -H "X-Secret-Key: &lt;secret&gt;" \
-d '{"max_bytes": null, "max_objects": null}'</code></pre>
<div class="alert alert-light border mb-0">
<div class="d-flex gap-2">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-muted mt-1" viewBox="0 0 16 16">
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
</svg>
<div>
<strong>Version Counting:</strong> When versioning is enabled, archived versions count toward the quota (similar to MinIO behavior). The quota is checked against total storage, not just current objects.
</div>
</div>
</div>
</div>
</article>
<article id="encryption" class="card shadow-sm docs-section"> <article id="encryption" class="card shadow-sm docs-section">
<div class="card-body"> <div class="card-body">
<div class="d-flex align-items-center gap-2 mb-3"> <div class="d-flex align-items-center gap-2 mb-3">
<span class="docs-section-kicker">09</span> <span class="docs-section-kicker">11</span>
<h2 class="h4 mb-0">Encryption</h2> <h2 class="h4 mb-0">Encryption</h2>
</div> </div>
<p class="text-muted">Protect data at rest with server-side encryption using AES-256-GCM. Objects are encrypted before being written to disk and decrypted transparently on read.</p> <p class="text-muted">Protect data at rest with server-side encryption using AES-256-GCM. Objects are encrypted before being written to disk and decrypted transparently on read.</p>
@@ -515,7 +583,7 @@ curl -X DELETE "{{ api_base }}/kms/keys/{key-id}?waiting_period_days=30" \
<article id="troubleshooting" class="card shadow-sm docs-section"> <article id="troubleshooting" class="card shadow-sm docs-section">
<div class="card-body"> <div class="card-body">
<div class="d-flex align-items-center gap-2 mb-3"> <div class="d-flex align-items-center gap-2 mb-3">
<span class="docs-section-kicker">10</span> <span class="docs-section-kicker">12</span>
<h2 class="h4 mb-0">Troubleshooting &amp; tips</h2> <h2 class="h4 mb-0">Troubleshooting &amp; tips</h2>
</div> </div>
<div class="table-responsive"> <div class="table-responsive">
@@ -572,6 +640,7 @@ curl -X DELETE "{{ api_base }}/kms/keys/{key-id}?waiting_period_days=30" \
<li><a href="#api">REST endpoints</a></li> <li><a href="#api">REST endpoints</a></li>
<li><a href="#examples">API Examples</a></li> <li><a href="#examples">API Examples</a></li>
<li><a href="#replication">Site Replication</a></li> <li><a href="#replication">Site Replication</a></li>
<li><a href="#quotas">Bucket Quotas</a></li>
<li><a href="#encryption">Encryption</a></li> <li><a href="#encryption">Encryption</a></li>
<li><a href="#troubleshooting">Troubleshooting</a></li> <li><a href="#troubleshooting">Troubleshooting</a></li>
</ul> </ul>