Compare commits
2 Commits
453ac6ea30
...
f160827b41
| Author | SHA1 | Date | |
|---|---|---|---|
| f160827b41 | |||
| 9368715b16 |
@@ -86,7 +86,7 @@ Presigned URLs follow the AWS CLI playbook:
|
|||||||
| `AWS_REGION` | `us-east-1` | Region used in Signature V4 scope |
|
| `AWS_REGION` | `us-east-1` | Region used in Signature V4 scope |
|
||||||
| `AWS_SERVICE` | `s3` | Service used in Signature V4 scope |
|
| `AWS_SERVICE` | `s3` | Service used in Signature V4 scope |
|
||||||
|
|
||||||
> Buckets now live directly under `data/` while system metadata (versions, IAM, bucket policies, multipart uploads, etc.) lives in `data/.myfsio.sys`. Existing installs can keep their environment variables, but the defaults now match MinIO's `data/.system` pattern for easier bind-mounting.
|
> Buckets now live directly under `data/` while system metadata (versions, IAM, bucket policies, multipart uploads, etc.) lives in `data/.myfsio.sys`.
|
||||||
|
|
||||||
## API Cheatsheet (IAM headers required)
|
## API Cheatsheet (IAM headers required)
|
||||||
|
|
||||||
|
|||||||
@@ -266,5 +266,11 @@ class EncryptedObjectStorage:
|
|||||||
def list_multipart_parts(self, bucket_name: str, upload_id: str):
|
def list_multipart_parts(self, bucket_name: str, upload_id: str):
|
||||||
return self.storage.list_multipart_parts(bucket_name, upload_id)
|
return self.storage.list_multipart_parts(bucket_name, upload_id)
|
||||||
|
|
||||||
|
def get_bucket_quota(self, bucket_name: str):
|
||||||
|
return self.storage.get_bucket_quota(bucket_name)
|
||||||
|
|
||||||
|
def set_bucket_quota(self, bucket_name: str, *, max_bytes=None, max_objects=None):
|
||||||
|
return self.storage.set_bucket_quota(bucket_name, max_bytes=max_bytes, max_objects=max_objects)
|
||||||
|
|
||||||
def _compute_etag(self, path: Path) -> str:
|
def _compute_etag(self, path: Path) -> str:
|
||||||
return self.storage._compute_etag(path)
|
return self.storage._compute_etag(path)
|
||||||
|
|||||||
@@ -129,6 +129,25 @@ class EntityTooLargeError(AppError):
|
|||||||
status_code: int = 413
|
status_code: int = 413
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class QuotaExceededAppError(AppError):
|
||||||
|
"""Bucket quota exceeded."""
|
||||||
|
code: str = "QuotaExceeded"
|
||||||
|
message: str = "The bucket quota has been exceeded"
|
||||||
|
status_code: int = 403
|
||||||
|
quota: Optional[Dict[str, Any]] = None
|
||||||
|
usage: Optional[Dict[str, int]] = None
|
||||||
|
|
||||||
|
def __post_init__(self):
|
||||||
|
if self.quota or self.usage:
|
||||||
|
self.details = {}
|
||||||
|
if self.quota:
|
||||||
|
self.details["quota"] = self.quota
|
||||||
|
if self.usage:
|
||||||
|
self.details["usage"] = self.usage
|
||||||
|
super().__post_init__()
|
||||||
|
|
||||||
|
|
||||||
def handle_app_error(error: AppError) -> Response:
|
def handle_app_error(error: AppError) -> Response:
|
||||||
"""Handle application errors with appropriate response format."""
|
"""Handle application errors with appropriate response format."""
|
||||||
log_extra = {"error_code": error.code}
|
log_extra = {"error_code": error.code}
|
||||||
@@ -163,5 +182,6 @@ def register_error_handlers(app):
|
|||||||
ObjectNotFoundError, InvalidObjectKeyError,
|
ObjectNotFoundError, InvalidObjectKeyError,
|
||||||
AccessDeniedError, InvalidCredentialsError,
|
AccessDeniedError, InvalidCredentialsError,
|
||||||
MalformedRequestError, InvalidArgumentError, EntityTooLargeError,
|
MalformedRequestError, InvalidArgumentError, EntityTooLargeError,
|
||||||
|
QuotaExceededAppError,
|
||||||
]:
|
]:
|
||||||
app.register_error_handler(error_class, handle_app_error)
|
app.register_error_handler(error_class, handle_app_error)
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ from .bucket_policies import BucketPolicyStore
|
|||||||
from .extensions import limiter
|
from .extensions import limiter
|
||||||
from .iam import IamError, Principal
|
from .iam import IamError, Principal
|
||||||
from .replication import ReplicationManager
|
from .replication import ReplicationManager
|
||||||
from .storage import ObjectStorage, StorageError
|
from .storage import ObjectStorage, StorageError, QuotaExceededError
|
||||||
|
|
||||||
s3_api_bp = Blueprint("s3_api", __name__)
|
s3_api_bp = Blueprint("s3_api", __name__)
|
||||||
|
|
||||||
@@ -803,6 +803,7 @@ def _maybe_handle_bucket_subresource(bucket_name: str) -> Response | None:
|
|||||||
"acl": _bucket_acl_handler,
|
"acl": _bucket_acl_handler,
|
||||||
"versions": _bucket_list_versions_handler,
|
"versions": _bucket_list_versions_handler,
|
||||||
"lifecycle": _bucket_lifecycle_handler,
|
"lifecycle": _bucket_lifecycle_handler,
|
||||||
|
"quota": _bucket_quota_handler,
|
||||||
}
|
}
|
||||||
requested = [key for key in handlers if key in request.args]
|
requested = [key for key in handlers if key in request.args]
|
||||||
if not requested:
|
if not requested:
|
||||||
@@ -1400,6 +1401,87 @@ def _parse_lifecycle_config(payload: bytes) -> list:
|
|||||||
return rules
|
return rules
|
||||||
|
|
||||||
|
|
||||||
|
def _bucket_quota_handler(bucket_name: str) -> Response:
|
||||||
|
"""Handle bucket quota configuration (GET/PUT/DELETE /<bucket>?quota)."""
|
||||||
|
if request.method not in {"GET", "PUT", "DELETE"}:
|
||||||
|
return _method_not_allowed(["GET", "PUT", "DELETE"])
|
||||||
|
|
||||||
|
principal, error = _require_principal()
|
||||||
|
if error:
|
||||||
|
return error
|
||||||
|
try:
|
||||||
|
_authorize_action(principal, bucket_name, "policy")
|
||||||
|
except IamError as exc:
|
||||||
|
return _error_response("AccessDenied", str(exc), 403)
|
||||||
|
|
||||||
|
storage = _storage()
|
||||||
|
|
||||||
|
if not storage.bucket_exists(bucket_name):
|
||||||
|
return _error_response("NoSuchBucket", "Bucket does not exist", 404)
|
||||||
|
|
||||||
|
if request.method == "GET":
|
||||||
|
quota = storage.get_bucket_quota(bucket_name)
|
||||||
|
if not quota:
|
||||||
|
return _error_response("NoSuchQuotaConfiguration", "No quota configuration found", 404)
|
||||||
|
|
||||||
|
# Return as JSON for simplicity (not a standard S3 API)
|
||||||
|
stats = storage.bucket_stats(bucket_name)
|
||||||
|
return jsonify({
|
||||||
|
"quota": quota,
|
||||||
|
"usage": {
|
||||||
|
"bytes": stats.get("bytes", 0),
|
||||||
|
"objects": stats.get("objects", 0),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
if request.method == "DELETE":
|
||||||
|
try:
|
||||||
|
storage.set_bucket_quota(bucket_name, max_size_bytes=None, max_objects=None)
|
||||||
|
except StorageError as exc:
|
||||||
|
return _error_response("NoSuchBucket", str(exc), 404)
|
||||||
|
current_app.logger.info("Bucket quota deleted", extra={"bucket": bucket_name})
|
||||||
|
return Response(status=204)
|
||||||
|
|
||||||
|
# PUT
|
||||||
|
payload = request.get_json(silent=True)
|
||||||
|
if not payload:
|
||||||
|
return _error_response("MalformedRequest", "Request body must be JSON with quota limits", 400)
|
||||||
|
|
||||||
|
max_size_bytes = payload.get("max_size_bytes")
|
||||||
|
max_objects = payload.get("max_objects")
|
||||||
|
|
||||||
|
if max_size_bytes is None and max_objects is None:
|
||||||
|
return _error_response("InvalidArgument", "At least one of max_size_bytes or max_objects is required", 400)
|
||||||
|
|
||||||
|
# Validate types
|
||||||
|
if max_size_bytes is not None:
|
||||||
|
try:
|
||||||
|
max_size_bytes = int(max_size_bytes)
|
||||||
|
if max_size_bytes < 0:
|
||||||
|
raise ValueError("must be non-negative")
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
return _error_response("InvalidArgument", f"max_size_bytes {exc}", 400)
|
||||||
|
|
||||||
|
if max_objects is not None:
|
||||||
|
try:
|
||||||
|
max_objects = int(max_objects)
|
||||||
|
if max_objects < 0:
|
||||||
|
raise ValueError("must be non-negative")
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
return _error_response("InvalidArgument", f"max_objects {exc}", 400)
|
||||||
|
|
||||||
|
try:
|
||||||
|
storage.set_bucket_quota(bucket_name, max_size_bytes=max_size_bytes, max_objects=max_objects)
|
||||||
|
except StorageError as exc:
|
||||||
|
return _error_response("NoSuchBucket", str(exc), 404)
|
||||||
|
|
||||||
|
current_app.logger.info(
|
||||||
|
"Bucket quota updated",
|
||||||
|
extra={"bucket": bucket_name, "max_size_bytes": max_size_bytes, "max_objects": max_objects}
|
||||||
|
)
|
||||||
|
return Response(status=204)
|
||||||
|
|
||||||
|
|
||||||
def _bulk_delete_handler(bucket_name: str) -> Response:
|
def _bulk_delete_handler(bucket_name: str) -> Response:
|
||||||
principal, error = _require_principal()
|
principal, error = _require_principal()
|
||||||
if error:
|
if error:
|
||||||
@@ -1749,6 +1831,8 @@ def object_handler(bucket_name: str, object_key: str):
|
|||||||
stream,
|
stream,
|
||||||
metadata=metadata or None,
|
metadata=metadata or None,
|
||||||
)
|
)
|
||||||
|
except QuotaExceededError as exc:
|
||||||
|
return _error_response("QuotaExceeded", str(exc), 403)
|
||||||
except StorageError as exc:
|
except StorageError as exc:
|
||||||
message = str(exc)
|
message = str(exc)
|
||||||
if "Bucket" in message:
|
if "Bucket" in message:
|
||||||
@@ -2256,6 +2340,8 @@ def _complete_multipart_upload(bucket_name: str, object_key: str) -> Response:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
meta = _storage().complete_multipart_upload(bucket_name, upload_id, parts)
|
meta = _storage().complete_multipart_upload(bucket_name, upload_id, parts)
|
||||||
|
except QuotaExceededError as exc:
|
||||||
|
return _error_response("QuotaExceeded", str(exc), 403)
|
||||||
except StorageError as exc:
|
except StorageError as exc:
|
||||||
if "NoSuchBucket" in str(exc):
|
if "NoSuchBucket" in str(exc):
|
||||||
return _error_response("NoSuchBucket", str(exc), 404)
|
return _error_response("NoSuchBucket", str(exc), 404)
|
||||||
|
|||||||
234
app/storage.py
234
app/storage.py
@@ -75,6 +75,15 @@ class StorageError(RuntimeError):
|
|||||||
"""Raised when the storage layer encounters an unrecoverable problem."""
|
"""Raised when the storage layer encounters an unrecoverable problem."""
|
||||||
|
|
||||||
|
|
||||||
|
class QuotaExceededError(StorageError):
|
||||||
|
"""Raised when an operation would exceed bucket quota limits."""
|
||||||
|
|
||||||
|
def __init__(self, message: str, quota: Dict[str, Any], usage: Dict[str, int]):
|
||||||
|
super().__init__(message)
|
||||||
|
self.quota = quota
|
||||||
|
self.usage = usage
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ObjectMeta:
|
class ObjectMeta:
|
||||||
key: str
|
key: str
|
||||||
@@ -169,16 +178,38 @@ class ObjectStorage:
|
|||||||
|
|
||||||
object_count = 0
|
object_count = 0
|
||||||
total_bytes = 0
|
total_bytes = 0
|
||||||
|
version_count = 0
|
||||||
|
version_bytes = 0
|
||||||
|
|
||||||
|
# Count current objects in the bucket folder
|
||||||
for path in bucket_path.rglob("*"):
|
for path in bucket_path.rglob("*"):
|
||||||
if path.is_file():
|
if path.is_file():
|
||||||
rel = path.relative_to(bucket_path)
|
rel = path.relative_to(bucket_path)
|
||||||
if rel.parts and rel.parts[0] in self.INTERNAL_FOLDERS:
|
if not rel.parts:
|
||||||
continue
|
continue
|
||||||
stat = path.stat()
|
top_folder = rel.parts[0]
|
||||||
object_count += 1
|
if top_folder not in self.INTERNAL_FOLDERS:
|
||||||
total_bytes += stat.st_size
|
stat = path.stat()
|
||||||
|
object_count += 1
|
||||||
|
total_bytes += stat.st_size
|
||||||
|
|
||||||
stats = {"objects": object_count, "bytes": total_bytes}
|
# Count archived versions in the system folder
|
||||||
|
versions_root = self._bucket_versions_root(bucket_name)
|
||||||
|
if versions_root.exists():
|
||||||
|
for path in versions_root.rglob("*.bin"):
|
||||||
|
if path.is_file():
|
||||||
|
stat = path.stat()
|
||||||
|
version_count += 1
|
||||||
|
version_bytes += stat.st_size
|
||||||
|
|
||||||
|
stats = {
|
||||||
|
"objects": object_count,
|
||||||
|
"bytes": total_bytes,
|
||||||
|
"version_count": version_count,
|
||||||
|
"version_bytes": version_bytes,
|
||||||
|
"total_objects": object_count + version_count, # All objects including versions
|
||||||
|
"total_bytes": total_bytes + version_bytes, # All storage including versions
|
||||||
|
}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
@@ -243,6 +274,7 @@ class ObjectStorage:
|
|||||||
stream: BinaryIO,
|
stream: BinaryIO,
|
||||||
*,
|
*,
|
||||||
metadata: Optional[Dict[str, str]] = None,
|
metadata: Optional[Dict[str, str]] = None,
|
||||||
|
enforce_quota: bool = True,
|
||||||
) -> ObjectMeta:
|
) -> ObjectMeta:
|
||||||
bucket_path = self._bucket_path(bucket_name)
|
bucket_path = self._bucket_path(bucket_name)
|
||||||
if not bucket_path.exists():
|
if not bucket_path.exists():
|
||||||
@@ -253,12 +285,52 @@ class ObjectStorage:
|
|||||||
destination = bucket_path / safe_key
|
destination = bucket_path / safe_key
|
||||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
if self._is_versioning_enabled(bucket_path) and destination.exists():
|
# Check if this is an overwrite (won't add to object count)
|
||||||
|
is_overwrite = destination.exists()
|
||||||
|
existing_size = destination.stat().st_size if is_overwrite else 0
|
||||||
|
|
||||||
|
if self._is_versioning_enabled(bucket_path) and is_overwrite:
|
||||||
self._archive_current_version(bucket_id, safe_key, reason="overwrite")
|
self._archive_current_version(bucket_id, safe_key, reason="overwrite")
|
||||||
|
|
||||||
checksum = hashlib.md5()
|
# Write to temp file first to get actual size
|
||||||
with destination.open("wb") as target:
|
tmp_dir = self._system_root_path() / self.SYSTEM_TMP_DIR
|
||||||
shutil.copyfileobj(_HashingReader(stream, checksum), target)
|
tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
tmp_path = tmp_dir / f"{uuid.uuid4().hex}.tmp"
|
||||||
|
|
||||||
|
try:
|
||||||
|
checksum = hashlib.md5()
|
||||||
|
with tmp_path.open("wb") as target:
|
||||||
|
shutil.copyfileobj(_HashingReader(stream, checksum), target)
|
||||||
|
|
||||||
|
new_size = tmp_path.stat().st_size
|
||||||
|
|
||||||
|
# Check quota before finalizing
|
||||||
|
if enforce_quota:
|
||||||
|
# Calculate net change (new size minus size being replaced)
|
||||||
|
size_delta = new_size - existing_size
|
||||||
|
object_delta = 0 if is_overwrite else 1
|
||||||
|
|
||||||
|
quota_check = self.check_quota(
|
||||||
|
bucket_name,
|
||||||
|
additional_bytes=max(0, size_delta),
|
||||||
|
additional_objects=object_delta,
|
||||||
|
)
|
||||||
|
if not quota_check["allowed"]:
|
||||||
|
raise QuotaExceededError(
|
||||||
|
quota_check["message"] or "Quota exceeded",
|
||||||
|
quota_check["quota"],
|
||||||
|
quota_check["usage"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Move to final destination
|
||||||
|
shutil.move(str(tmp_path), str(destination))
|
||||||
|
|
||||||
|
finally:
|
||||||
|
# Clean up temp file if it still exists
|
||||||
|
try:
|
||||||
|
tmp_path.unlink(missing_ok=True)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
stat = destination.stat()
|
stat = destination.stat()
|
||||||
if metadata:
|
if metadata:
|
||||||
@@ -424,6 +496,124 @@ class ObjectStorage:
|
|||||||
bucket_path = self._require_bucket_path(bucket_name)
|
bucket_path = self._require_bucket_path(bucket_name)
|
||||||
self._set_bucket_config_entry(bucket_path.name, "lifecycle", rules)
|
self._set_bucket_config_entry(bucket_path.name, "lifecycle", rules)
|
||||||
|
|
||||||
|
def get_bucket_quota(self, bucket_name: str) -> Dict[str, Any]:
|
||||||
|
"""Get quota configuration for bucket.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with 'max_bytes' and 'max_objects' (None if unlimited).
|
||||||
|
"""
|
||||||
|
bucket_path = self._require_bucket_path(bucket_name)
|
||||||
|
config = self._read_bucket_config(bucket_path.name)
|
||||||
|
quota = config.get("quota")
|
||||||
|
if isinstance(quota, dict):
|
||||||
|
return {
|
||||||
|
"max_bytes": quota.get("max_bytes"),
|
||||||
|
"max_objects": quota.get("max_objects"),
|
||||||
|
}
|
||||||
|
return {"max_bytes": None, "max_objects": None}
|
||||||
|
|
||||||
|
def set_bucket_quota(
|
||||||
|
self,
|
||||||
|
bucket_name: str,
|
||||||
|
*,
|
||||||
|
max_bytes: Optional[int] = None,
|
||||||
|
max_objects: Optional[int] = None,
|
||||||
|
) -> None:
|
||||||
|
"""Set quota limits for a bucket.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
bucket_name: Name of the bucket
|
||||||
|
max_bytes: Maximum total size in bytes (None to remove limit)
|
||||||
|
max_objects: Maximum number of objects (None to remove limit)
|
||||||
|
"""
|
||||||
|
bucket_path = self._require_bucket_path(bucket_name)
|
||||||
|
|
||||||
|
if max_bytes is None and max_objects is None:
|
||||||
|
# Remove quota entirely
|
||||||
|
self._set_bucket_config_entry(bucket_path.name, "quota", None)
|
||||||
|
return
|
||||||
|
|
||||||
|
quota: Dict[str, Any] = {}
|
||||||
|
if max_bytes is not None:
|
||||||
|
if max_bytes < 0:
|
||||||
|
raise StorageError("max_bytes must be non-negative")
|
||||||
|
quota["max_bytes"] = max_bytes
|
||||||
|
if max_objects is not None:
|
||||||
|
if max_objects < 0:
|
||||||
|
raise StorageError("max_objects must be non-negative")
|
||||||
|
quota["max_objects"] = max_objects
|
||||||
|
|
||||||
|
self._set_bucket_config_entry(bucket_path.name, "quota", quota)
|
||||||
|
|
||||||
|
def check_quota(
|
||||||
|
self,
|
||||||
|
bucket_name: str,
|
||||||
|
additional_bytes: int = 0,
|
||||||
|
additional_objects: int = 0,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Check if an operation would exceed bucket quota.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
bucket_name: Name of the bucket
|
||||||
|
additional_bytes: Bytes that would be added
|
||||||
|
additional_objects: Objects that would be added
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with 'allowed' (bool), 'quota' (current limits),
|
||||||
|
'usage' (current usage), and 'message' (if not allowed).
|
||||||
|
"""
|
||||||
|
quota = self.get_bucket_quota(bucket_name)
|
||||||
|
if not quota:
|
||||||
|
return {
|
||||||
|
"allowed": True,
|
||||||
|
"quota": None,
|
||||||
|
"usage": None,
|
||||||
|
"message": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Get current stats (uses cache when available)
|
||||||
|
stats = self.bucket_stats(bucket_name)
|
||||||
|
# Use totals which include versions for quota enforcement
|
||||||
|
current_bytes = stats.get("total_bytes", stats.get("bytes", 0))
|
||||||
|
current_objects = stats.get("total_objects", stats.get("objects", 0))
|
||||||
|
|
||||||
|
result = {
|
||||||
|
"allowed": True,
|
||||||
|
"quota": quota,
|
||||||
|
"usage": {
|
||||||
|
"bytes": current_bytes,
|
||||||
|
"objects": current_objects,
|
||||||
|
"version_count": stats.get("version_count", 0),
|
||||||
|
"version_bytes": stats.get("version_bytes", 0),
|
||||||
|
},
|
||||||
|
"message": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
max_bytes_limit = quota.get("max_bytes")
|
||||||
|
max_objects = quota.get("max_objects")
|
||||||
|
|
||||||
|
if max_bytes_limit is not None:
|
||||||
|
projected_bytes = current_bytes + additional_bytes
|
||||||
|
if projected_bytes > max_bytes_limit:
|
||||||
|
result["allowed"] = False
|
||||||
|
result["message"] = (
|
||||||
|
f"Quota exceeded: adding {additional_bytes} bytes would result in "
|
||||||
|
f"{projected_bytes} bytes, exceeding limit of {max_bytes_limit} bytes"
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
if max_objects is not None:
|
||||||
|
projected_objects = current_objects + additional_objects
|
||||||
|
if projected_objects > max_objects:
|
||||||
|
result["allowed"] = False
|
||||||
|
result["message"] = (
|
||||||
|
f"Quota exceeded: adding {additional_objects} objects would result in "
|
||||||
|
f"{projected_objects} objects, exceeding limit of {max_objects} objects"
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
def get_object_tags(self, bucket_name: str, object_key: str) -> List[Dict[str, str]]:
|
def get_object_tags(self, bucket_name: str, object_key: str) -> List[Dict[str, str]]:
|
||||||
"""Get tags for an object."""
|
"""Get tags for an object."""
|
||||||
bucket_path = self._bucket_path(bucket_name)
|
bucket_path = self._bucket_path(bucket_name)
|
||||||
@@ -540,6 +730,7 @@ class ObjectStorage:
|
|||||||
else:
|
else:
|
||||||
self._delete_metadata(bucket_id, safe_key)
|
self._delete_metadata(bucket_id, safe_key)
|
||||||
stat = destination.stat()
|
stat = destination.stat()
|
||||||
|
self._invalidate_bucket_stats_cache(bucket_id)
|
||||||
return ObjectMeta(
|
return ObjectMeta(
|
||||||
key=safe_key.as_posix(),
|
key=safe_key.as_posix(),
|
||||||
size=stat.st_size,
|
size=stat.st_size,
|
||||||
@@ -688,6 +879,7 @@ class ObjectStorage:
|
|||||||
bucket_name: str,
|
bucket_name: str,
|
||||||
upload_id: str,
|
upload_id: str,
|
||||||
ordered_parts: List[Dict[str, Any]],
|
ordered_parts: List[Dict[str, Any]],
|
||||||
|
enforce_quota: bool = True,
|
||||||
) -> ObjectMeta:
|
) -> ObjectMeta:
|
||||||
if not ordered_parts:
|
if not ordered_parts:
|
||||||
raise StorageError("parts list required")
|
raise StorageError("parts list required")
|
||||||
@@ -698,6 +890,7 @@ class ObjectStorage:
|
|||||||
if not parts_map:
|
if not parts_map:
|
||||||
raise StorageError("No uploaded parts found")
|
raise StorageError("No uploaded parts found")
|
||||||
validated: List[tuple[int, Dict[str, Any]]] = []
|
validated: List[tuple[int, Dict[str, Any]]] = []
|
||||||
|
total_size = 0
|
||||||
for part in ordered_parts:
|
for part in ordered_parts:
|
||||||
raw_number = part.get("part_number")
|
raw_number = part.get("part_number")
|
||||||
if raw_number is None:
|
if raw_number is None:
|
||||||
@@ -717,10 +910,33 @@ class ObjectStorage:
|
|||||||
if supplied_etag and record.get("etag") and supplied_etag.strip('"') != record["etag"]:
|
if supplied_etag and record.get("etag") and supplied_etag.strip('"') != record["etag"]:
|
||||||
raise StorageError(f"ETag mismatch for part {number}")
|
raise StorageError(f"ETag mismatch for part {number}")
|
||||||
validated.append((number, record))
|
validated.append((number, record))
|
||||||
|
total_size += record.get("size", 0)
|
||||||
validated.sort(key=lambda entry: entry[0])
|
validated.sort(key=lambda entry: entry[0])
|
||||||
|
|
||||||
safe_key = self._sanitize_object_key(manifest["object_key"])
|
safe_key = self._sanitize_object_key(manifest["object_key"])
|
||||||
destination = bucket_path / safe_key
|
destination = bucket_path / safe_key
|
||||||
|
|
||||||
|
# Check if this is an overwrite
|
||||||
|
is_overwrite = destination.exists()
|
||||||
|
existing_size = destination.stat().st_size if is_overwrite else 0
|
||||||
|
|
||||||
|
# Check quota before writing
|
||||||
|
if enforce_quota:
|
||||||
|
size_delta = total_size - existing_size
|
||||||
|
object_delta = 0 if is_overwrite else 1
|
||||||
|
|
||||||
|
quota_check = self.check_quota(
|
||||||
|
bucket_name,
|
||||||
|
additional_bytes=max(0, size_delta),
|
||||||
|
additional_objects=object_delta,
|
||||||
|
)
|
||||||
|
if not quota_check["allowed"]:
|
||||||
|
raise QuotaExceededError(
|
||||||
|
quota_check["message"] or "Quota exceeded",
|
||||||
|
quota_check["quota"],
|
||||||
|
quota_check["usage"],
|
||||||
|
)
|
||||||
|
|
||||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
lock_file_path = self._system_bucket_root(bucket_id) / "locks" / f"{safe_key.as_posix().replace('/', '_')}.lock"
|
lock_file_path = self._system_bucket_root(bucket_id) / "locks" / f"{safe_key.as_posix().replace('/', '_')}.lock"
|
||||||
|
|||||||
92
app/ui.py
92
app/ui.py
@@ -260,9 +260,9 @@ def buckets_overview():
|
|||||||
visible_buckets.append({
|
visible_buckets.append({
|
||||||
"meta": bucket,
|
"meta": bucket,
|
||||||
"summary": {
|
"summary": {
|
||||||
"objects": stats["objects"],
|
"objects": stats["total_objects"],
|
||||||
"total_bytes": stats["bytes"],
|
"total_bytes": stats["total_bytes"],
|
||||||
"human_size": _format_bytes(stats["bytes"]),
|
"human_size": _format_bytes(stats["total_bytes"]),
|
||||||
},
|
},
|
||||||
"access_label": access_label,
|
"access_label": access_label,
|
||||||
"access_badge": access_badge,
|
"access_badge": access_badge,
|
||||||
@@ -372,6 +372,16 @@ def bucket_detail(bucket_name: str):
|
|||||||
encryption_enabled = current_app.config.get("ENCRYPTION_ENABLED", False)
|
encryption_enabled = current_app.config.get("ENCRYPTION_ENABLED", False)
|
||||||
can_manage_encryption = can_manage_versioning # Same as other bucket properties
|
can_manage_encryption = can_manage_versioning # Same as other bucket properties
|
||||||
|
|
||||||
|
# Quota settings (admin only)
|
||||||
|
bucket_quota = storage.get_bucket_quota(bucket_name)
|
||||||
|
bucket_stats = storage.bucket_stats(bucket_name)
|
||||||
|
can_manage_quota = False
|
||||||
|
try:
|
||||||
|
_iam().authorize(principal, None, "iam:list_users")
|
||||||
|
can_manage_quota = True
|
||||||
|
except IamError:
|
||||||
|
pass
|
||||||
|
|
||||||
return render_template(
|
return render_template(
|
||||||
"bucket_detail.html",
|
"bucket_detail.html",
|
||||||
bucket_name=bucket_name,
|
bucket_name=bucket_name,
|
||||||
@@ -392,6 +402,9 @@ def bucket_detail(bucket_name: str):
|
|||||||
kms_keys=kms_keys,
|
kms_keys=kms_keys,
|
||||||
kms_enabled=kms_enabled,
|
kms_enabled=kms_enabled,
|
||||||
encryption_enabled=encryption_enabled,
|
encryption_enabled=encryption_enabled,
|
||||||
|
bucket_quota=bucket_quota,
|
||||||
|
bucket_stats=bucket_stats,
|
||||||
|
can_manage_quota=can_manage_quota,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -925,6 +938,71 @@ def update_bucket_versioning(bucket_name: str):
|
|||||||
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
|
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
|
||||||
|
|
||||||
|
|
||||||
|
@ui_bp.post("/buckets/<bucket_name>/quota")
|
||||||
|
def update_bucket_quota(bucket_name: str):
|
||||||
|
"""Update bucket quota configuration (admin only)."""
|
||||||
|
principal = _current_principal()
|
||||||
|
|
||||||
|
# Quota management is admin-only
|
||||||
|
is_admin = False
|
||||||
|
try:
|
||||||
|
_iam().authorize(principal, None, "iam:list_users")
|
||||||
|
is_admin = True
|
||||||
|
except IamError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if not is_admin:
|
||||||
|
flash("Only administrators can manage bucket quotas", "danger")
|
||||||
|
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
|
||||||
|
|
||||||
|
action = request.form.get("action", "set")
|
||||||
|
|
||||||
|
if action == "remove":
|
||||||
|
try:
|
||||||
|
_storage().set_bucket_quota(bucket_name, max_bytes=None, max_objects=None)
|
||||||
|
flash("Bucket quota removed", "info")
|
||||||
|
except StorageError as exc:
|
||||||
|
flash(_friendly_error_message(exc), "danger")
|
||||||
|
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
|
||||||
|
|
||||||
|
# Parse quota values
|
||||||
|
max_mb_str = request.form.get("max_mb", "").strip()
|
||||||
|
max_objects_str = request.form.get("max_objects", "").strip()
|
||||||
|
|
||||||
|
max_bytes = None
|
||||||
|
max_objects = None
|
||||||
|
|
||||||
|
if max_mb_str:
|
||||||
|
try:
|
||||||
|
max_mb = int(max_mb_str)
|
||||||
|
if max_mb < 1:
|
||||||
|
raise ValueError("Size must be at least 1 MB")
|
||||||
|
max_bytes = max_mb * 1024 * 1024 # Convert MB to bytes
|
||||||
|
except ValueError as exc:
|
||||||
|
flash(f"Invalid size value: {exc}", "danger")
|
||||||
|
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
|
||||||
|
|
||||||
|
if max_objects_str:
|
||||||
|
try:
|
||||||
|
max_objects = int(max_objects_str)
|
||||||
|
if max_objects < 0:
|
||||||
|
raise ValueError("Object count must be non-negative")
|
||||||
|
except ValueError as exc:
|
||||||
|
flash(f"Invalid object count: {exc}", "danger")
|
||||||
|
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
|
||||||
|
|
||||||
|
try:
|
||||||
|
_storage().set_bucket_quota(bucket_name, max_bytes=max_bytes, max_objects=max_objects)
|
||||||
|
if max_bytes is None and max_objects is None:
|
||||||
|
flash("Bucket quota removed", "info")
|
||||||
|
else:
|
||||||
|
flash("Bucket quota updated", "success")
|
||||||
|
except StorageError as exc:
|
||||||
|
flash(_friendly_error_message(exc), "danger")
|
||||||
|
|
||||||
|
return redirect(url_for("ui.bucket_detail", bucket_name=bucket_name, tab="properties"))
|
||||||
|
|
||||||
|
|
||||||
@ui_bp.post("/buckets/<bucket_name>/encryption")
|
@ui_bp.post("/buckets/<bucket_name>/encryption")
|
||||||
def update_bucket_encryption(bucket_name: str):
|
def update_bucket_encryption(bucket_name: str):
|
||||||
"""Update bucket default encryption configuration."""
|
"""Update bucket default encryption configuration."""
|
||||||
@@ -1438,13 +1516,16 @@ def metrics_dashboard():
|
|||||||
|
|
||||||
total_objects = 0
|
total_objects = 0
|
||||||
total_bytes_used = 0
|
total_bytes_used = 0
|
||||||
|
total_versions = 0
|
||||||
|
|
||||||
# Note: Uses cached stats from storage layer to improve performance
|
# Note: Uses cached stats from storage layer to improve performance
|
||||||
cache_ttl = current_app.config.get("BUCKET_STATS_CACHE_TTL", 60)
|
cache_ttl = current_app.config.get("BUCKET_STATS_CACHE_TTL", 60)
|
||||||
for bucket in buckets:
|
for bucket in buckets:
|
||||||
stats = storage.bucket_stats(bucket.name, cache_ttl=cache_ttl)
|
stats = storage.bucket_stats(bucket.name, cache_ttl=cache_ttl)
|
||||||
total_objects += stats["objects"]
|
# Use totals which include archived versions
|
||||||
total_bytes_used += stats["bytes"]
|
total_objects += stats.get("total_objects", stats.get("objects", 0))
|
||||||
|
total_bytes_used += stats.get("total_bytes", stats.get("bytes", 0))
|
||||||
|
total_versions += stats.get("version_count", 0)
|
||||||
|
|
||||||
return render_template(
|
return render_template(
|
||||||
"metrics.html",
|
"metrics.html",
|
||||||
@@ -1465,6 +1546,7 @@ def metrics_dashboard():
|
|||||||
app={
|
app={
|
||||||
"buckets": total_buckets,
|
"buckets": total_buckets,
|
||||||
"objects": total_objects,
|
"objects": total_objects,
|
||||||
|
"versions": total_versions,
|
||||||
"storage_used": _format_bytes(total_bytes_used),
|
"storage_used": _format_bytes(total_bytes_used),
|
||||||
"storage_raw": total_bytes_used,
|
"storage_raw": total_bytes_used,
|
||||||
}
|
}
|
||||||
|
|||||||
76
docs.md
76
docs.md
@@ -340,7 +340,71 @@ To verify an object is encrypted:
|
|||||||
2. Look for `.meta` files containing encryption metadata
|
2. Look for `.meta` files containing encryption metadata
|
||||||
3. Download via the API/UI - the object should be automatically decrypted
|
3. Download via the API/UI - the object should be automatically decrypted
|
||||||
|
|
||||||
## 8. Site Replication
|
## 8. Bucket Quotas
|
||||||
|
|
||||||
|
MyFSIO supports **storage quotas** to limit how much data a bucket can hold. Quotas are enforced on uploads and multipart completions.
|
||||||
|
|
||||||
|
### Quota Types
|
||||||
|
|
||||||
|
| Limit | Description |
|
||||||
|
|-------|-------------|
|
||||||
|
| **Max Size (MB)** | Maximum total storage in megabytes (includes current objects + archived versions) |
|
||||||
|
| **Max Objects** | Maximum number of objects (includes current objects + archived versions) |
|
||||||
|
|
||||||
|
### Managing Quotas (Admin Only)
|
||||||
|
|
||||||
|
Quota management is restricted to administrators (users with `iam:*` or `iam:list_users` permissions).
|
||||||
|
|
||||||
|
#### Via UI
|
||||||
|
|
||||||
|
1. Navigate to your bucket in the UI
|
||||||
|
2. Click the **Properties** tab
|
||||||
|
3. Find the **Storage Quota** card
|
||||||
|
4. Enter limits:
|
||||||
|
- **Max Size (MB)**: Leave empty for unlimited
|
||||||
|
- **Max Objects**: Leave empty for unlimited
|
||||||
|
5. Click **Update Quota**
|
||||||
|
|
||||||
|
To remove a quota, click **Remove Quota**.
|
||||||
|
|
||||||
|
#### Via API
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Set quota (max 100MB, max 1000 objects)
|
||||||
|
curl -X PUT "http://localhost:5000/bucket/<bucket>?quota" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "X-Access-Key: ..." -H "X-Secret-Key: ..." \
|
||||||
|
-d '{"max_bytes": 104857600, "max_objects": 1000}'
|
||||||
|
|
||||||
|
# Get current quota
|
||||||
|
curl "http://localhost:5000/bucket/<bucket>?quota" \
|
||||||
|
-H "X-Access-Key: ..." -H "X-Secret-Key: ..."
|
||||||
|
|
||||||
|
# Remove quota
|
||||||
|
curl -X PUT "http://localhost:5000/bucket/<bucket>?quota" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "X-Access-Key: ..." -H "X-Secret-Key: ..." \
|
||||||
|
-d '{"max_bytes": null, "max_objects": null}'
|
||||||
|
```
|
||||||
|
|
||||||
|
### Quota Behavior
|
||||||
|
|
||||||
|
- **Version Counting**: When versioning is enabled, archived versions count toward the quota
|
||||||
|
- **Enforcement Points**: Quotas are checked during `PUT` object and `CompleteMultipartUpload` operations
|
||||||
|
- **Error Response**: When quota is exceeded, the API returns `HTTP 400` with error code `QuotaExceeded`
|
||||||
|
- **Visibility**: All users can view quota usage in the bucket detail page, but only admins can modify quotas
|
||||||
|
|
||||||
|
### Example Error
|
||||||
|
|
||||||
|
```xml
|
||||||
|
<Error>
|
||||||
|
<Code>QuotaExceeded</Code>
|
||||||
|
<Message>Bucket quota exceeded: storage limit reached</Message>
|
||||||
|
<BucketName>my-bucket</BucketName>
|
||||||
|
</Error>
|
||||||
|
```
|
||||||
|
|
||||||
|
## 9. Site Replication
|
||||||
|
|
||||||
### Permission Model
|
### Permission Model
|
||||||
|
|
||||||
@@ -477,7 +541,7 @@ To set up two-way replication (Server A ↔ Server B):
|
|||||||
|
|
||||||
**Note**: Deleting a bucket will automatically remove its associated replication configuration.
|
**Note**: Deleting a bucket will automatically remove its associated replication configuration.
|
||||||
|
|
||||||
## 9. Running Tests
|
## 11. Running Tests
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pytest -q
|
pytest -q
|
||||||
@@ -487,7 +551,7 @@ The suite now includes a boto3 integration test that spins up a live HTTP server
|
|||||||
|
|
||||||
The suite covers bucket CRUD, presigned downloads, bucket policy enforcement, and regression tests for anonymous reads when a Public policy is attached.
|
The suite covers bucket CRUD, presigned downloads, bucket policy enforcement, and regression tests for anonymous reads when a Public policy is attached.
|
||||||
|
|
||||||
## 10. Troubleshooting
|
## 12. Troubleshooting
|
||||||
|
|
||||||
| Symptom | Likely Cause | Fix |
|
| Symptom | Likely Cause | Fix |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
@@ -496,7 +560,7 @@ The suite covers bucket CRUD, presigned downloads, bucket policy enforcement, an
|
|||||||
| Presign modal errors with 403 | IAM user lacks `read/write/delete` for target bucket or bucket policy denies | Update IAM inline policies or remove conflicting deny statements. |
|
| Presign modal errors with 403 | IAM user lacks `read/write/delete` for target bucket or bucket policy denies | Update IAM inline policies or remove conflicting deny statements. |
|
||||||
| Large upload rejected immediately | File exceeds `MAX_UPLOAD_SIZE` | Increase env var or shrink object. |
|
| Large upload rejected immediately | File exceeds `MAX_UPLOAD_SIZE` | Increase env var or shrink object. |
|
||||||
|
|
||||||
## 11. API Matrix
|
## 13. API Matrix
|
||||||
|
|
||||||
```
|
```
|
||||||
GET / # List buckets
|
GET / # List buckets
|
||||||
@@ -510,9 +574,11 @@ POST /presign/<bucket>/<key> # Generate SigV4 URL
|
|||||||
GET /bucket-policy/<bucket> # Fetch policy
|
GET /bucket-policy/<bucket> # Fetch policy
|
||||||
PUT /bucket-policy/<bucket> # Upsert policy
|
PUT /bucket-policy/<bucket> # Upsert policy
|
||||||
DELETE /bucket-policy/<bucket> # Delete policy
|
DELETE /bucket-policy/<bucket> # Delete policy
|
||||||
|
GET /<bucket>?quota # Get bucket quota
|
||||||
|
PUT /<bucket>?quota # Set bucket quota (admin only)
|
||||||
```
|
```
|
||||||
|
|
||||||
## 12. Next Steps
|
## 14. Next Steps
|
||||||
|
|
||||||
- Tailor IAM + policy JSON files for team-ready presets.
|
- Tailor IAM + policy JSON files for team-ready presets.
|
||||||
- Wrap `run_api.py` with gunicorn or another WSGI server for long-running workloads.
|
- Wrap `run_api.py` with gunicorn or another WSGI server for long-running workloads.
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
Flask>=3.0.2
|
Flask>=3.1.2
|
||||||
Flask-Limiter>=3.5.0
|
Flask-Limiter>=4.1.0
|
||||||
Flask-Cors>=4.0.0
|
Flask-Cors>=6.0.1
|
||||||
Flask-WTF>=1.2.1
|
Flask-WTF>=1.2.2
|
||||||
pytest>=7.4
|
pytest>=9.0.1
|
||||||
requests>=2.31
|
requests>=2.32.5
|
||||||
boto3>=1.34
|
boto3>=1.42.1
|
||||||
waitress>=2.1.2
|
waitress>=3.0.2
|
||||||
psutil>=5.9.0
|
psutil>=7.1.3
|
||||||
cryptography>=41.0.0
|
cryptography>=46.0.3
|
||||||
@@ -730,6 +730,158 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
<!-- Storage Quota Card -->
|
||||||
|
<div class="card shadow-sm mt-4" id="bucket-quota-card">
|
||||||
|
<div class="card-header d-flex align-items-center">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" fill="currentColor" class="text-primary me-2" viewBox="0 0 16 16">
|
||||||
|
<path d="M1 2.5A1.5 1.5 0 0 1 2.5 1h3A1.5 1.5 0 0 1 7 2.5v3A1.5 1.5 0 0 1 5.5 7h-3A1.5 1.5 0 0 1 1 5.5v-3zM2.5 2a.5.5 0 0 0-.5.5v3a.5.5 0 0 0 .5.5h3a.5.5 0 0 0 .5-.5v-3a.5.5 0 0 0-.5-.5h-3zm6.5.5A1.5 1.5 0 0 1 10.5 1h3A1.5 1.5 0 0 1 15 2.5v3A1.5 1.5 0 0 1 13.5 7h-3A1.5 1.5 0 0 1 9 5.5v-3zm1.5-.5a.5.5 0 0 0-.5.5v3a.5.5 0 0 0 .5.5h3a.5.5 0 0 0 .5-.5v-3a.5.5 0 0 0-.5-.5h-3zM1 10.5A1.5 1.5 0 0 1 2.5 9h3A1.5 1.5 0 0 1 7 10.5v3A1.5 1.5 0 0 1 5.5 15h-3A1.5 1.5 0 0 1 1 13.5v-3zm1.5-.5a.5.5 0 0 0-.5.5v3a.5.5 0 0 0 .5.5h3a.5.5 0 0 0 .5-.5v-3a.5.5 0 0 0-.5-.5h-3zm6.5.5A1.5 1.5 0 0 1 10.5 9h3a1.5 1.5 0 0 1 1.5 1.5v3a1.5 1.5 0 0 1-1.5 1.5h-3A1.5 1.5 0 0 1 9 13.5v-3zm1.5-.5a.5.5 0 0 0-.5.5v3a.5.5 0 0 0 .5.5h3a.5.5 0 0 0 .5-.5v-3a.5.5 0 0 0-.5-.5h-3z"/>
|
||||||
|
</svg>
|
||||||
|
<span class="fw-semibold">Storage Quota</span>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
{% set max_bytes = bucket_quota.get('max_bytes') %}
|
||||||
|
{% set max_objects = bucket_quota.get('max_objects') %}
|
||||||
|
{% set has_quota = max_bytes is not none or max_objects is not none %}
|
||||||
|
{% set current_objects = bucket_stats.get('objects', 0) %}
|
||||||
|
{% set version_count = bucket_stats.get('version_count', 0) %}
|
||||||
|
{% set total_objects = bucket_stats.get('total_objects', current_objects) %}
|
||||||
|
{% set current_bytes = bucket_stats.get('bytes', 0) %}
|
||||||
|
{% set version_bytes = bucket_stats.get('version_bytes', 0) %}
|
||||||
|
{% set total_bytes = bucket_stats.get('total_bytes', current_bytes) %}
|
||||||
|
|
||||||
|
<!-- Current Usage Display -->
|
||||||
|
<div class="mb-4">
|
||||||
|
<h6 class="small fw-semibold mb-3">Current Usage</h6>
|
||||||
|
<div class="row g-3">
|
||||||
|
<div class="col-6">
|
||||||
|
<div class="border rounded p-3 text-center">
|
||||||
|
<div class="fs-4 fw-bold text-primary">{{ total_objects }}</div>
|
||||||
|
<div class="small text-muted">Total Objects</div>
|
||||||
|
{% if max_objects is not none %}
|
||||||
|
<div class="progress mt-2" style="height: 4px;">
|
||||||
|
{% set obj_pct = (total_objects / max_objects * 100) | int if max_objects > 0 else 0 %}
|
||||||
|
<div class="progress-bar {% if obj_pct >= 90 %}bg-danger{% elif obj_pct >= 75 %}bg-warning{% else %}bg-success{% endif %}" style="width: {{ [obj_pct, 100] | min }}%"></div>
|
||||||
|
</div>
|
||||||
|
<div class="small text-muted mt-1">{{ obj_pct }}% of {{ max_objects }} limit</div>
|
||||||
|
{% else %}
|
||||||
|
<div class="small text-muted mt-2">No limit</div>
|
||||||
|
{% endif %}
|
||||||
|
{% if version_count > 0 %}
|
||||||
|
<div class="small text-muted mt-1">
|
||||||
|
<span class="text-body-secondary">({{ current_objects }} current + {{ version_count }} versions)</span>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-6">
|
||||||
|
<div class="border rounded p-3 text-center">
|
||||||
|
<div class="fs-4 fw-bold text-primary">{{ total_bytes | filesizeformat }}</div>
|
||||||
|
<div class="small text-muted">Total Storage</div>
|
||||||
|
{% if max_bytes is not none %}
|
||||||
|
<div class="progress mt-2" style="height: 4px;">
|
||||||
|
{% set bytes_pct = (total_bytes / max_bytes * 100) | int if max_bytes > 0 else 0 %}
|
||||||
|
<div class="progress-bar {% if bytes_pct >= 90 %}bg-danger{% elif bytes_pct >= 75 %}bg-warning{% else %}bg-success{% endif %}" style="width: {{ [bytes_pct, 100] | min }}%"></div>
|
||||||
|
</div>
|
||||||
|
<div class="small text-muted mt-1">{{ bytes_pct }}% of {{ max_bytes | filesizeformat }} limit</div>
|
||||||
|
{% else %}
|
||||||
|
<div class="small text-muted mt-2">No limit</div>
|
||||||
|
{% endif %}
|
||||||
|
{% if version_bytes > 0 %}
|
||||||
|
<div class="small text-muted mt-1">
|
||||||
|
<span class="text-body-secondary">({{ current_bytes | filesizeformat }} current + {{ version_bytes | filesizeformat }} versions)</span>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{% if has_quota %}
|
||||||
|
<!-- Quota Enabled State -->
|
||||||
|
<div class="alert alert-info d-flex align-items-start mb-4" role="alert">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="me-2 flex-shrink-0" viewBox="0 0 16 16">
|
||||||
|
<path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm.93-9.412-1 4.705c-.07.34.029.533.304.533.194 0 .487-.07.686-.246l-.088.416c-.287.346-.92.598-1.465.598-.703 0-1.002-.422-.808-1.319l.738-3.468c.064-.293.006-.399-.287-.47l-.451-.081.082-.381 2.29-.287zM8 5.5a1 1 0 1 1 0-2 1 1 0 0 1 0 2z"/>
|
||||||
|
</svg>
|
||||||
|
<div>
|
||||||
|
<strong>Storage quota enabled</strong>
|
||||||
|
<p class="mb-0 small">
|
||||||
|
{% if max_bytes is not none and max_objects is not none %}
|
||||||
|
Limited to {{ max_bytes | filesizeformat }} and {{ max_objects }} objects.
|
||||||
|
{% elif max_bytes is not none %}
|
||||||
|
Limited to {{ max_bytes | filesizeformat }} storage.
|
||||||
|
{% else %}
|
||||||
|
Limited to {{ max_objects }} objects.
|
||||||
|
{% endif %}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% else %}
|
||||||
|
<!-- Quota Disabled State -->
|
||||||
|
<div class="alert alert-secondary d-flex align-items-start mb-4" role="alert">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="me-2 flex-shrink-0" viewBox="0 0 16 16">
|
||||||
|
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||||||
|
<path d="M4.646 4.646a.5.5 0 0 1 .708 0L8 7.293l2.646-2.647a.5.5 0 0 1 .708.708L8.707 8l2.647 2.646a.5.5 0 0 1-.708.708L8 8.707l-2.646 2.647a.5.5 0 0 1-.708-.708L7.293 8 4.646 5.354a.5.5 0 0 1 0-.708z"/>
|
||||||
|
</svg>
|
||||||
|
<div>
|
||||||
|
<strong>No storage quota</strong>
|
||||||
|
<p class="mb-0 small">This bucket has no storage or object count limits. Set limits below to control usage.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% if can_manage_quota %}
|
||||||
|
<form method="post" action="{{ url_for('ui.update_bucket_quota', bucket_name=bucket_name) }}" id="quotaForm">
|
||||||
|
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
|
||||||
|
|
||||||
|
<!-- Max Storage -->
|
||||||
|
<div class="mb-3">
|
||||||
|
<label for="max_mb" class="form-label fw-medium">Maximum Storage Size</label>
|
||||||
|
<div class="input-group">
|
||||||
|
<input type="number" class="form-control" id="max_mb" name="max_mb"
|
||||||
|
value="{{ (max_bytes / 1048576) | int if max_bytes is not none else '' }}"
|
||||||
|
min="1" step="1" placeholder="Unlimited">
|
||||||
|
<span class="input-group-text">MB</span>
|
||||||
|
</div>
|
||||||
|
<div class="form-text">Minimum 1 MB. Leave empty for unlimited.</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Max Objects -->
|
||||||
|
<div class="mb-4">
|
||||||
|
<label for="max_objects" class="form-label fw-medium">Maximum Object Count</label>
|
||||||
|
<input type="number" class="form-control" id="max_objects" name="max_objects"
|
||||||
|
value="{{ max_objects if max_objects is not none else '' }}"
|
||||||
|
min="0" step="1" placeholder="Unlimited">
|
||||||
|
<div class="form-text">Maximum number of objects allowed. Leave empty for unlimited.</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="d-flex gap-2 flex-wrap">
|
||||||
|
<button class="btn btn-primary" type="submit">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||||
|
<path d="M12.736 3.97a.733.733 0 0 1 1.047 0c.286.289.29.756.01 1.05L7.88 12.01a.733.733 0 0 1-1.065.02L3.217 8.384a.757.757 0 0 1 0-1.06.733.733 0 0 1 1.047 0l3.052 3.093 5.4-6.425a.247.247 0 0 1 .02-.022Z"/>
|
||||||
|
</svg>
|
||||||
|
Save Quota Settings
|
||||||
|
</button>
|
||||||
|
{% if has_quota %}
|
||||||
|
<button type="submit" class="btn btn-outline-danger" id="removeQuotaBtn" name="action" value="remove">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||||
|
<path d="M4.646 4.646a.5.5 0 0 1 .708 0L8 7.293l2.646-2.647a.5.5 0 0 1 .708.708L8.707 8l2.647 2.646a.5.5 0 0 1-.708.708L8 8.707l-2.646 2.647a.5.5 0 0 1-.708-.708L7.293 8 4.646 5.354a.5.5 0 0 1 0-.708z"/>
|
||||||
|
</svg>
|
||||||
|
Remove Quota
|
||||||
|
</button>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
{% else %}
|
||||||
|
<div class="text-center py-3">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" fill="currentColor" class="text-muted mb-2" viewBox="0 0 16 16">
|
||||||
|
<path d="M8 1a2 2 0 0 1 2 2v4H6V3a2 2 0 0 1 2-2zm3 6V3a3 3 0 0 0-6 0v4a2 2 0 0 0-2 2v5a2 2 0 0 0 2 2h6a2 2 0 0 0 2-2V9a2 2 0 0 0-2-2z"/>
|
||||||
|
</svg>
|
||||||
|
<p class="text-muted mb-0 small">You do not have permission to modify quota settings for this bucket.</p>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Sidebar -->
|
<!-- Sidebar -->
|
||||||
|
|||||||
@@ -418,10 +418,78 @@ s3.complete_multipart_upload(
|
|||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</article>
|
</article>
|
||||||
|
<article id="quotas" class="card shadow-sm docs-section">
|
||||||
|
<div class="card-body">
|
||||||
|
<div class="d-flex align-items-center gap-2 mb-3">
|
||||||
|
<span class="docs-section-kicker">10</span>
|
||||||
|
<h2 class="h4 mb-0">Bucket Quotas</h2>
|
||||||
|
</div>
|
||||||
|
<p class="text-muted">Limit how much data a bucket can hold using storage quotas. Quotas are enforced on uploads and multipart completions.</p>
|
||||||
|
|
||||||
|
<h3 class="h6 text-uppercase text-muted mt-4">Quota Types</h3>
|
||||||
|
<div class="table-responsive mb-3">
|
||||||
|
<table class="table table-sm table-bordered small">
|
||||||
|
<thead class="table-light">
|
||||||
|
<tr>
|
||||||
|
<th>Limit</th>
|
||||||
|
<th>Description</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td><strong>Max Size (MB)</strong></td>
|
||||||
|
<td>Maximum total storage in megabytes (includes current objects + archived versions)</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><strong>Max Objects</strong></td>
|
||||||
|
<td>Maximum number of objects (includes current objects + archived versions)</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h3 class="h6 text-uppercase text-muted mt-4">Managing Quotas (Admin Only)</h3>
|
||||||
|
<p class="small text-muted">Quota management is restricted to administrators (users with <code>iam:*</code> permissions).</p>
|
||||||
|
<ol class="docs-steps mb-3">
|
||||||
|
<li>Navigate to your bucket → <strong>Properties</strong> tab → <strong>Storage Quota</strong> card.</li>
|
||||||
|
<li>Enter limits: <strong>Max Size (MB)</strong> and/or <strong>Max Objects</strong>. Leave empty for unlimited.</li>
|
||||||
|
<li>Click <strong>Update Quota</strong> to save, or <strong>Remove Quota</strong> to clear limits.</li>
|
||||||
|
</ol>
|
||||||
|
|
||||||
|
<h3 class="h6 text-uppercase text-muted mt-4">API Usage</h3>
|
||||||
|
<pre class="mb-3"><code class="language-bash"># Set quota (max 100MB, max 1000 objects)
|
||||||
|
curl -X PUT "{{ api_base }}/bucket/<bucket>?quota" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||||
|
-d '{"max_bytes": 104857600, "max_objects": 1000}'
|
||||||
|
|
||||||
|
# Get current quota
|
||||||
|
curl "{{ api_base }}/bucket/<bucket>?quota" \
|
||||||
|
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||||||
|
|
||||||
|
# Remove quota
|
||||||
|
curl -X PUT "{{ api_base }}/bucket/<bucket>?quota" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||||
|
-d '{"max_bytes": null, "max_objects": null}'</code></pre>
|
||||||
|
|
||||||
|
<div class="alert alert-light border mb-0">
|
||||||
|
<div class="d-flex gap-2">
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-muted mt-1" viewBox="0 0 16 16">
|
||||||
|
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||||||
|
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
||||||
|
</svg>
|
||||||
|
<div>
|
||||||
|
<strong>Version Counting:</strong> When versioning is enabled, archived versions count toward the quota. The quota is checked against total storage, not just current objects.
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</article>
|
||||||
<article id="encryption" class="card shadow-sm docs-section">
|
<article id="encryption" class="card shadow-sm docs-section">
|
||||||
<div class="card-body">
|
<div class="card-body">
|
||||||
<div class="d-flex align-items-center gap-2 mb-3">
|
<div class="d-flex align-items-center gap-2 mb-3">
|
||||||
<span class="docs-section-kicker">09</span>
|
<span class="docs-section-kicker">11</span>
|
||||||
<h2 class="h4 mb-0">Encryption</h2>
|
<h2 class="h4 mb-0">Encryption</h2>
|
||||||
</div>
|
</div>
|
||||||
<p class="text-muted">Protect data at rest with server-side encryption using AES-256-GCM. Objects are encrypted before being written to disk and decrypted transparently on read.</p>
|
<p class="text-muted">Protect data at rest with server-side encryption using AES-256-GCM. Objects are encrypted before being written to disk and decrypted transparently on read.</p>
|
||||||
@@ -515,7 +583,7 @@ curl -X DELETE "{{ api_base }}/kms/keys/{key-id}?waiting_period_days=30" \
|
|||||||
<article id="troubleshooting" class="card shadow-sm docs-section">
|
<article id="troubleshooting" class="card shadow-sm docs-section">
|
||||||
<div class="card-body">
|
<div class="card-body">
|
||||||
<div class="d-flex align-items-center gap-2 mb-3">
|
<div class="d-flex align-items-center gap-2 mb-3">
|
||||||
<span class="docs-section-kicker">10</span>
|
<span class="docs-section-kicker">12</span>
|
||||||
<h2 class="h4 mb-0">Troubleshooting & tips</h2>
|
<h2 class="h4 mb-0">Troubleshooting & tips</h2>
|
||||||
</div>
|
</div>
|
||||||
<div class="table-responsive">
|
<div class="table-responsive">
|
||||||
@@ -572,6 +640,7 @@ curl -X DELETE "{{ api_base }}/kms/keys/{key-id}?waiting_period_days=30" \
|
|||||||
<li><a href="#api">REST endpoints</a></li>
|
<li><a href="#api">REST endpoints</a></li>
|
||||||
<li><a href="#examples">API Examples</a></li>
|
<li><a href="#examples">API Examples</a></li>
|
||||||
<li><a href="#replication">Site Replication</a></li>
|
<li><a href="#replication">Site Replication</a></li>
|
||||||
|
<li><a href="#quotas">Bucket Quotas</a></li>
|
||||||
<li><a href="#encryption">Encryption</a></li>
|
<li><a href="#encryption">Encryption</a></li>
|
||||||
<li><a href="#troubleshooting">Troubleshooting</a></li>
|
<li><a href="#troubleshooting">Troubleshooting</a></li>
|
||||||
</ul>
|
</ul>
|
||||||
|
|||||||
Reference in New Issue
Block a user