Add configurable rate limits for S3 API endpoints

app/config.py

@@ -10,6 +10,23 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Dict, Optional
 
+import psutil
+
+
+def _calculate_auto_threads() -> int:
+    cpu_count = psutil.cpu_count(logical=True) or 4
+    return max(1, min(cpu_count * 2, 64))
+
+
+def _calculate_auto_connection_limit() -> int:
+    available_mb = psutil.virtual_memory().available / (1024 * 1024)
+    calculated = int(available_mb / 5)
+    return max(20, min(calculated, 1000))
+
+
+def _calculate_auto_backlog(connection_limit: int) -> int:
+    return max(64, min(connection_limit * 2, 4096))
+
 
 def _validate_rate_limit(value: str) -> str:
     pattern = r"^\d+\s+per\s+(second|minute|hour|day)$"
@@ -63,6 +80,10 @@ class AppConfig:
     log_backup_count: int
     ratelimit_default: str
     ratelimit_storage_uri: str
+    ratelimit_list_buckets: str
+    ratelimit_bucket_ops: str
+    ratelimit_object_ops: str
+    ratelimit_head_ops: str
     cors_origins: list[str]
     cors_methods: list[str]
     cors_allow_headers: list[str]
@@ -94,6 +115,9 @@ class AppConfig:
     server_connection_limit: int
     server_backlog: int
     server_channel_timeout: int
+    server_threads_auto: bool
+    server_connection_limit_auto: bool
+    server_backlog_auto: bool
     site_sync_enabled: bool
     site_sync_interval_seconds: int
     site_sync_batch_size: int
@@ -171,6 +195,10 @@ class AppConfig:
         log_backup_count = int(_get("LOG_BACKUP_COUNT", 3))
         ratelimit_default = _validate_rate_limit(str(_get("RATE_LIMIT_DEFAULT", "200 per minute")))
         ratelimit_storage_uri = str(_get("RATE_LIMIT_STORAGE_URI", "memory://"))
+        ratelimit_list_buckets = _validate_rate_limit(str(_get("RATE_LIMIT_LIST_BUCKETS", "60 per minute")))
+        ratelimit_bucket_ops = _validate_rate_limit(str(_get("RATE_LIMIT_BUCKET_OPS", "120 per minute")))
+        ratelimit_object_ops = _validate_rate_limit(str(_get("RATE_LIMIT_OBJECT_OPS", "240 per minute")))
+        ratelimit_head_ops = _validate_rate_limit(str(_get("RATE_LIMIT_HEAD_OPS", "100 per minute")))
 
         def _csv(value: str, default: list[str]) -> list[str]:
             if not value:
@@ -200,9 +228,30 @@ class AppConfig:
         operation_metrics_interval_minutes = int(_get("OPERATION_METRICS_INTERVAL_MINUTES", 5))
         operation_metrics_retention_hours = int(_get("OPERATION_METRICS_RETENTION_HOURS", 24))
 
-        server_threads = int(_get("SERVER_THREADS", 4))
+        _raw_threads = int(_get("SERVER_THREADS", 0))
-        server_connection_limit = int(_get("SERVER_CONNECTION_LIMIT", 100))
+        if _raw_threads == 0:
-        server_backlog = int(_get("SERVER_BACKLOG", 1024))
+            server_threads = _calculate_auto_threads()
+            server_threads_auto = True
+        else:
+            server_threads = _raw_threads
+            server_threads_auto = False
+
+        _raw_conn_limit = int(_get("SERVER_CONNECTION_LIMIT", 0))
+        if _raw_conn_limit == 0:
+            server_connection_limit = _calculate_auto_connection_limit()
+            server_connection_limit_auto = True
+        else:
+            server_connection_limit = _raw_conn_limit
+            server_connection_limit_auto = False
+
+        _raw_backlog = int(_get("SERVER_BACKLOG", 0))
+        if _raw_backlog == 0:
+            server_backlog = _calculate_auto_backlog(server_connection_limit)
+            server_backlog_auto = True
+        else:
+            server_backlog = _raw_backlog
+            server_backlog_auto = False
+
         server_channel_timeout = int(_get("SERVER_CHANNEL_TIMEOUT", 120))
         site_sync_enabled = str(_get("SITE_SYNC_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
         site_sync_interval_seconds = int(_get("SITE_SYNC_INTERVAL_SECONDS", 60))
@@ -225,6 +274,10 @@ class AppConfig:
                    log_backup_count=log_backup_count,
                    ratelimit_default=ratelimit_default,
                    ratelimit_storage_uri=ratelimit_storage_uri,
+                   ratelimit_list_buckets=ratelimit_list_buckets,
+                   ratelimit_bucket_ops=ratelimit_bucket_ops,
+                   ratelimit_object_ops=ratelimit_object_ops,
+                   ratelimit_head_ops=ratelimit_head_ops,
                    cors_origins=cors_origins,
                    cors_methods=cors_methods,
                    cors_allow_headers=cors_allow_headers,
@@ -256,6 +309,9 @@ class AppConfig:
                    server_connection_limit=server_connection_limit,
                    server_backlog=server_backlog,
                    server_channel_timeout=server_channel_timeout,
+                   server_threads_auto=server_threads_auto,
+                   server_connection_limit_auto=server_connection_limit_auto,
+                   server_backlog_auto=server_backlog_auto,
                    site_sync_enabled=site_sync_enabled,
                    site_sync_interval_seconds=site_sync_interval_seconds,
                    site_sync_batch_size=site_sync_batch_size)
@@ -364,9 +420,11 @@ class AppConfig:
             print(f"  ENCRYPTION:       Enabled (Master key: {self.encryption_master_key_path})")
         if self.kms_enabled:
             print(f"  KMS:              Enabled (Keys: {self.kms_keys_path})")
-        print(f"  SERVER_THREADS:   {self.server_threads}")
+        def _auto(flag: bool) -> str:
-        print(f"  CONNECTION_LIMIT: {self.server_connection_limit}")
+            return " (auto)" if flag else ""
-        print(f"  BACKLOG:          {self.server_backlog}")
+        print(f"  SERVER_THREADS:   {self.server_threads}{_auto(self.server_threads_auto)}")
+        print(f"  CONNECTION_LIMIT: {self.server_connection_limit}{_auto(self.server_connection_limit_auto)}")
+        print(f"  BACKLOG:          {self.server_backlog}{_auto(self.server_backlog_auto)}")
         print(f"  CHANNEL_TIMEOUT:  {self.server_channel_timeout}s")
         print("=" * 60)
         
@@ -406,6 +464,10 @@ class AppConfig:
             "LOG_BACKUP_COUNT": self.log_backup_count,
             "RATELIMIT_DEFAULT": self.ratelimit_default,
             "RATELIMIT_STORAGE_URI": self.ratelimit_storage_uri,
+            "RATELIMIT_LIST_BUCKETS": self.ratelimit_list_buckets,
+            "RATELIMIT_BUCKET_OPS": self.ratelimit_bucket_ops,
+            "RATELIMIT_OBJECT_OPS": self.ratelimit_object_ops,
+            "RATELIMIT_HEAD_OPS": self.ratelimit_head_ops,
             "CORS_ORIGINS": self.cors_origins,
             "CORS_METHODS": self.cors_methods,
             "CORS_ALLOW_HEADERS": self.cors_allow_headers,

app/s3_api.py

@@ -82,6 +82,22 @@ def _access_logging() -> AccessLoggingService:
     return current_app.extensions["access_logging"]
 
 
+def _get_list_buckets_limit() -> str:
+    return current_app.config.get("RATELIMIT_LIST_BUCKETS", "60 per minute")
+
+
+def _get_bucket_ops_limit() -> str:
+    return current_app.config.get("RATELIMIT_BUCKET_OPS", "120 per minute")
+
+
+def _get_object_ops_limit() -> str:
+    return current_app.config.get("RATELIMIT_OBJECT_OPS", "240 per minute")
+
+
+def _get_head_ops_limit() -> str:
+    return current_app.config.get("RATELIMIT_HEAD_OPS", "100 per minute")
+
+
 def _xml_response(element: Element, status: int = 200) -> Response:
     xml_bytes = tostring(element, encoding="utf-8")
     return Response(xml_bytes, status=status, mimetype="application/xml")
@@ -2143,7 +2159,7 @@ def _bulk_delete_handler(bucket_name: str) -> Response:
 
 
 @s3_api_bp.get("/")
-@limiter.limit("60 per minute")
+@limiter.limit(_get_list_buckets_limit)
 def list_buckets() -> Response:
     principal, error = _require_principal()
     if error:
@@ -2171,7 +2187,7 @@ def list_buckets() -> Response:
 
 
 @s3_api_bp.route("/<bucket_name>", methods=["PUT", "DELETE", "GET", "POST"], strict_slashes=False)
-@limiter.limit("120 per minute")
+@limiter.limit(_get_bucket_ops_limit)
 def bucket_handler(bucket_name: str) -> Response:
     storage = _storage()
     subresource_response = _maybe_handle_bucket_subresource(bucket_name)
@@ -2363,7 +2379,7 @@ def bucket_handler(bucket_name: str) -> Response:
 
 
 @s3_api_bp.route("/<bucket_name>/<path:object_key>", methods=["PUT", "GET", "DELETE", "HEAD", "POST"], strict_slashes=False)
-@limiter.limit("240 per minute")
+@limiter.limit(_get_object_ops_limit)
 def object_handler(bucket_name: str, object_key: str):
     storage = _storage()
 
@@ -2681,7 +2697,7 @@ def _bucket_policy_handler(bucket_name: str) -> Response:
 
 
 @s3_api_bp.route("/<bucket_name>", methods=["HEAD"])
-@limiter.limit("100 per minute")
+@limiter.limit(_get_head_ops_limit)
 def head_bucket(bucket_name: str) -> Response:
     principal, error = _require_principal()
     if error:
@@ -2696,7 +2712,7 @@ def head_bucket(bucket_name: str) -> Response:
 
 
 @s3_api_bp.route("/<bucket_name>/<path:object_key>", methods=["HEAD"])
-@limiter.limit("100 per minute")
+@limiter.limit(_get_head_ops_limit)
 def head_object(bucket_name: str, object_key: str) -> Response:
     principal, error = _require_principal()
     if error:

app/version.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-APP_VERSION = "0.2.3"
+APP_VERSION = "0.2.4"
 
 
 def get_version() -> str:

docs.md

@@ -166,15 +166,19 @@ All configuration is done via environment variables. The table below lists every
 | Variable | Default | Notes |
 | --- | --- | --- |
 | `RATE_LIMIT_DEFAULT` | `200 per minute` | Default rate limit for API endpoints. |
+| `RATE_LIMIT_LIST_BUCKETS` | `60 per minute` | Rate limit for listing buckets (`GET /`). |
+| `RATE_LIMIT_BUCKET_OPS` | `120 per minute` | Rate limit for bucket operations (PUT/DELETE/GET/POST on `/<bucket>`). |
+| `RATE_LIMIT_OBJECT_OPS` | `240 per minute` | Rate limit for object operations (PUT/GET/DELETE/POST on `/<bucket>/<key>`). |
+| `RATE_LIMIT_HEAD_OPS` | `100 per minute` | Rate limit for HEAD requests (bucket and object). |
 | `RATE_LIMIT_STORAGE_URI` | `memory://` | Storage backend for rate limits. Use `redis://host:port` for distributed setups. |
 
 ### Server Configuration
 
 | Variable | Default | Notes |
 | --- | --- | --- |
-| `SERVER_THREADS` | `4` | Waitress worker threads (1-64). More threads handle more concurrent requests but use more memory. |
+| `SERVER_THREADS` | `0` (auto) | Waitress worker threads (1-64). Set to `0` for auto-calculation based on CPU cores (×2). |
-| `SERVER_CONNECTION_LIMIT` | `100` | Maximum concurrent connections (10-1000). Ensure OS file descriptor limits support this value. |
+| `SERVER_CONNECTION_LIMIT` | `0` (auto) | Maximum concurrent connections (10-1000). Set to `0` for auto-calculation based on available RAM. |
-| `SERVER_BACKLOG` | `1024` | TCP listen backlog (64-4096). Connections queue here when all threads are busy. |
+| `SERVER_BACKLOG` | `0` (auto) | TCP listen backlog (64-4096). Set to `0` for auto-calculation (connection_limit × 2). |
 | `SERVER_CHANNEL_TIMEOUT` | `120` | Seconds before idle connections are closed (10-300). |
 
 ### Logging

templates/docs.html

@@ -157,23 +157,43 @@ python run.py --mode ui
                 <td><code>200 per minute</code></td>
                 <td>Default API rate limit.</td>
               </tr>
+              <tr>
+                <td><code>RATE_LIMIT_LIST_BUCKETS</code></td>
+                <td><code>60 per minute</code></td>
+                <td>Rate limit for listing buckets.</td>
+              </tr>
+              <tr>
+                <td><code>RATE_LIMIT_BUCKET_OPS</code></td>
+                <td><code>120 per minute</code></td>
+                <td>Rate limit for bucket operations.</td>
+              </tr>
+              <tr>
+                <td><code>RATE_LIMIT_OBJECT_OPS</code></td>
+                <td><code>240 per minute</code></td>
+                <td>Rate limit for object operations.</td>
+              </tr>
+              <tr>
+                <td><code>RATE_LIMIT_HEAD_OPS</code></td>
+                <td><code>100 per minute</code></td>
+                <td>Rate limit for HEAD requests.</td>
+              </tr>
               <tr class="table-secondary">
                 <td colspan="3" class="fw-semibold">Server Settings</td>
               </tr>
               <tr>
                 <td><code>SERVER_THREADS</code></td>
-                <td><code>4</code></td>
+                <td><code>0</code> (auto)</td>
-                <td>Waitress worker threads (1-64).</td>
+                <td>Waitress worker threads (1-64). 0 = auto (CPU cores × 2).</td>
               </tr>
               <tr>
                 <td><code>SERVER_CONNECTION_LIMIT</code></td>
-                <td><code>100</code></td>
+                <td><code>0</code> (auto)</td>
-                <td>Max concurrent connections (10-1000).</td>
+                <td>Max concurrent connections (10-1000). 0 = auto (RAM-based).</td>
               </tr>
               <tr>
                 <td><code>SERVER_BACKLOG</code></td>
-                <td><code>1024</code></td>
+                <td><code>0</code> (auto)</td>
-                <td>TCP listen backlog (64-4096).</td>
+                <td>TCP listen backlog (64-4096). 0 = auto (conn_limit × 2).</td>
               </tr>
               <tr>
                 <td><code>SERVER_CHANNEL_TIMEOUT</code></td>